Mercurial > p > roundup > code
changeset 2089:93f03c6714d8
A few big changes in this commit:
1. The current indexer has been moved to backends/indexer_dbm in
anticipation of my writing an indexer_rdbms,
2. Changed indexer invocation during create / set to follow the pattern
set by the metakit backend, which was much cleaner, and
3. The "content" property of FileClass is now mutable in all but the
metakit backend.
Metakit needs to be changed to support the editing of "content". Hey, and
I learnt today that the metakit backend implements its own indexer. How
about that... :)
| author | Richard Jones <richard@users.sourceforge.net> |
|---|---|
| date | Fri, 19 Mar 2004 04:47:59 +0000 |
| parents | 90769be53b4b |
| children | ad1ef1d9f5a1 |
| files | CHANGES.txt roundup/backends/back_anydbm.py roundup/backends/back_metakit.py roundup/backends/back_mysql.py roundup/backends/back_postgresql.py roundup/backends/blobfiles.py roundup/backends/indexer_dbm.py roundup/backends/rdbms_common.py roundup/backends/sessions_dbm.py roundup/indexer.py roundup/roundupdb.py test/db_test_base.py test/session_common.py test/test_indexer.py test/test_mailgw.py |
| diffstat | 15 files changed, 576 insertions(+), 488 deletions(-) [+] |
line wrap: on
line diff
--- a/CHANGES.txt Thu Mar 18 22:47:00 2004 +0000 +++ b/CHANGES.txt Fri Mar 19 04:47:59 2004 +0000 @@ -55,6 +55,7 @@ - the mail gateway now searches recursively for the text/plain and the attachments of a message (sf bug 841241). - fixed display of feedback messages in some situations (sf bug 739545) +- fixed ability to edit "content" property (sf bug 914062) Cleanup: - replace curuserid attribute on Database with the extended getuid() method.
--- a/roundup/backends/back_anydbm.py Thu Mar 18 22:47:00 2004 +0000 +++ b/roundup/backends/back_anydbm.py Fri Mar 19 04:47:59 2004 +0000 @@ -15,7 +15,7 @@ # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. # -#$Id: back_anydbm.py,v 1.138 2004-03-18 01:58:45 richard Exp $ +#$Id: back_anydbm.py,v 1.139 2004-03-19 04:47:59 richard Exp $ '''This module defines a backend that saves the hyperdatabase in a database chosen by anydbm. It is guaranteed to always be available in python versions >2.1.1 (the dumbdbm fallback in 2.1.1 and earlier has several @@ -37,7 +37,7 @@ from roundup import hyperdb, date, password, roundupdb, security from blobfiles import FileStorage from sessions_dbm import Sessions, OneTimeKeys -from roundup.indexer import Indexer +from indexer_dbm import Indexer from roundup.backends import locking from roundup.hyperdb import String, Password, Date, Interval, Link, \ Multilink, DatabaseError, Boolean, Number, Node @@ -882,6 +882,7 @@ elif isinstance(prop, String): if type(value) != type('') and type(value) != type(u''): raise TypeError, 'new property "%s" not a string'%key + self.db.indexer.add_text((self.classname, newid, key), value) elif isinstance(prop, Password): if not isinstance(value, password.Password): @@ -1143,6 +1144,15 @@ These operations trigger detectors and can be vetoed. Attempts to modify the "creation" or "activity" properties cause a KeyError. ''' + self.fireAuditors('set', nodeid, propvalues) + oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid)) + propvalues = self.set_inner(nodeid, **propvalues) + self.fireReactors('set', nodeid, oldvalues) + return propvalues + + def set_inner(self, nodeid, **propvalues): + ''' Called by set, in-between the audit and react calls. + ''' if not propvalues: return propvalues @@ -1155,11 +1165,6 @@ if self.db.journaltag is None: raise DatabaseError, 'Database open read-only' - self.fireAuditors('set', nodeid, propvalues) - # Take a copy of the node dict so that the subsequent set - # operation doesn't modify the oldvalues structure. - oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid)) - node = self.db.getnode(self.classname, nodeid) if node.has_key(self.db.RETIRED_FLAG): raise IndexError @@ -1290,6 +1295,8 @@ elif isinstance(prop, String): if value is not None and type(value) != type('') and type(value) != type(u''): raise TypeError, 'new property "%s" not a string'%propname + self.db.indexer.add_text((self.classname, nodeid, propname), + value) elif isinstance(prop, Password): if not isinstance(value, password.Password): @@ -1331,9 +1338,7 @@ if self.do_journal: self.db.addjournal(self.classname, nodeid, 'set', journalvalues) - self.fireReactors('set', nodeid, oldvalues) - - return propvalues + return propvalues def retire(self, nodeid): '''Retire a node. @@ -1946,20 +1951,18 @@ self.properties.update(properties) def index(self, nodeid): - '''Add (or refresh) the node to search indexes - ''' + ''' Add (or refresh) the node to search indexes ''' # find all the String properties that have indexme for prop, propclass in self.getprops().items(): - if isinstance(propclass, String) and propclass.indexme: + if isinstance(propclass, hyperdb.String) and propclass.indexme: + # index them under (classname, nodeid, property) try: value = str(self.get(nodeid, prop)) except IndexError: - # node no longer exists - entry should be removed - self.db.indexer.purge_entry((self.classname, nodeid, prop)) - else: - # and index them under (classname, nodeid, property) - self.db.indexer.add_text((self.classname, nodeid, prop), - value) + # node has been destroyed + continue + self.db.indexer.add_text((self.classname, nodeid, prop), value) + # # Detector interface @@ -2012,8 +2015,15 @@ content = propvalues['content'] del propvalues['content'] + # make sure we have a MIME type + mime_type = propvalues.get('type', self.default_mime_type) + # do the database create - newid = Class.create_inner(self, **propvalues) + newid = self.create_inner(**propvalues) + + # and index! + self.db.indexer.add_text((self.classname, newid, 'content'), content, + mime_type) # fire reactors self.fireReactors('create', newid, None) @@ -2059,6 +2069,35 @@ else: return Class.get(self, nodeid, propname) + def set(self, itemid, **propvalues): + ''' Snarf the "content" propvalue and update it in a file + ''' + self.fireAuditors('set', itemid, propvalues) + oldvalues = copy.deepcopy(self.db.getnode(self.classname, itemid)) + + # now remove the content property so it's not stored in the db + content = None + if propvalues.has_key('content'): + content = propvalues['content'] + del propvalues['content'] + + # do the database create + propvalues = self.set_inner(itemid, **propvalues) + + # do content? + if content: + # store and index + self.db.storefile(self.classname, itemid, None, content) + mime_type = propvalues.get('type', self.get(itemid, 'type')) + if not mime_type: + mime_type = self.default_mime_type + self.db.indexer.add_text((self.classname, itemid, 'content'), + content, mime_type) + + # fire reactors + self.fireReactors('set', itemid, oldvalues) + return propvalues + def getprops(self, protected=1): ''' In addition to the actual properties on the node, these methods provide the "content" property. If the "protected" flag is true, @@ -2069,27 +2108,6 @@ d['content'] = hyperdb.String() return d - def index(self, nodeid): - ''' Index the node in the search index. - - We want to index the content in addition to the normal String - property indexing. - ''' - # perform normal indexing - Class.index(self, nodeid) - - # get the content to index - content = self.get(nodeid, 'content') - - # figure the mime type - if self.properties.has_key('type'): - mime_type = self.get(nodeid, 'type') - else: - mime_type = self.default_mime_type - - # and index! - self.db.indexer.add_text((self.classname, nodeid, 'content'), content, - mime_type) # deviation from spec - was called ItemClass class IssueClass(Class, roundupdb.IssueClass):
--- a/roundup/backends/back_metakit.py Thu Mar 18 22:47:00 2004 +0000 +++ b/roundup/backends/back_metakit.py Fri Mar 19 04:47:59 2004 +0000 @@ -1,4 +1,4 @@ -# $Id: back_metakit.py,v 1.62 2004-03-18 01:58:45 richard Exp $ +# $Id: back_metakit.py,v 1.63 2004-03-19 04:47:59 richard Exp $ '''Metakit backend for Roundup, originally by Gordon McMillan. Known Current Bugs: @@ -45,7 +45,7 @@ import metakit from sessions_dbm import Sessions, OneTimeKeys import re, marshal, os, sys, time, calendar -from roundup import indexer +from indexer_dbm import Indexer import locking from roundup.date import Range @@ -1783,7 +1783,7 @@ CURVERSION = 2 -class Indexer(indexer.Indexer): +class Indexer(Indexer): disallows = {'THE':1, 'THIS':1, 'ZZZ':1, 'THAT':1, 'WITH':1} def __init__(self, path, datadb): self.path = os.path.join(path, 'index.mk4')
--- a/roundup/backends/back_mysql.py Thu Mar 18 22:47:00 2004 +0000 +++ b/roundup/backends/back_mysql.py Fri Mar 19 04:47:59 2004 +0000 @@ -244,6 +244,7 @@ '%s_%s_l_idx'%(classname, ml), '%s_%s_n_idx'%(classname, ml) ] + table_name = '%s_%s'%(classname, ml) for index_name in l: if not self.sql_index_exists(table_name, index_name): continue
--- a/roundup/backends/back_postgresql.py Thu Mar 18 22:47:00 2004 +0000 +++ b/roundup/backends/back_postgresql.py Fri Mar 19 04:47:59 2004 +0000 @@ -146,30 +146,26 @@ cols, mls = self.determine_columns(spec.properties.items()) cols.append('id') cols.append('__retired__') - scols = ',' . join(['"%s" VARCHAR(255)' % x for x in cols]) + scols = ',' . join(['"%s" VARCHAR(255)'%x for x in cols]) sql = 'CREATE TABLE "_%s" (%s)' % (spec.classname, scols) - if __debug__: - print >>hyperdb.DEBUG, 'create_class', (self, sql) - + print >>hyperdb.DEBUG, 'create_class_table', (self, sql) self.cursor.execute(sql) self.create_class_table_indexes(spec) return cols, mls def create_journal_table(self, spec): - cols = ',' . join(['"%s" VARCHAR(255)' % x - for x in 'nodeid date tag action params' . split()]) + cols = ',' . join(['"%s" VARCHAR(255)'%x + for x in 'nodeid date tag action params' . split()]) sql = 'CREATE TABLE "%s__journal" (%s)'%(spec.classname, cols) - if __debug__: - print >>hyperdb.DEBUG, 'create_class', (self, sql) - + print >>hyperdb.DEBUG, 'create_journal_table', (self, sql) self.cursor.execute(sql) self.create_journal_table_indexes(spec) def create_multilink_table(self, spec, ml): sql = '''CREATE TABLE "%s_%s" (linkid VARCHAR(255), - nodeid VARCHAR(255))''' % (spec.classname, ml) + nodeid VARCHAR(255))'''%(spec.classname, ml) if __debug__: print >>hyperdb.DEBUG, 'create_class', (self, sql)
--- a/roundup/backends/blobfiles.py Thu Mar 18 22:47:00 2004 +0000 +++ b/roundup/backends/blobfiles.py Fri Mar 19 04:47:59 2004 +0000 @@ -15,7 +15,7 @@ # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. # -#$Id: blobfiles.py,v 1.11 2004-02-11 23:55:09 richard Exp $ +#$Id: blobfiles.py,v 1.12 2004-03-19 04:47:59 richard Exp $ '''This module exports file storage for roundup backends. Files are stored into a directory hierarchy. ''' @@ -77,12 +77,14 @@ if not os.path.exists(os.path.dirname(name)): os.makedirs(os.path.dirname(name)) - # open the temp file for writing - open(name + '.tmp', 'wb').write(content) - - # save off the commit action - self.transactions.append((self.doStoreFile, (classname, nodeid, - property))) + # save to a temp file + name = name + '.tmp' + # make sure we don't register the rename action more than once + if not os.path.exists(name): + # save off the rename action + self.transactions.append((self.doStoreFile, (classname, nodeid, + property))) + open(name, 'wb').write(content) def getfile(self, classname, nodeid, property): '''Get the content of the file in the database. @@ -115,6 +117,11 @@ # determine the name of the file to write to name = self.filename(classname, nodeid, property) + # content is being updated (and some platforms, eg. win32, won't + # let us rename over the top of the old file) + if os.path.exists(name): + os.remove(name) + # the file is currently ".tmp" - move it to its real name to commit os.rename(name+".tmp", name)
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/roundup/backends/indexer_dbm.py Fri Mar 19 04:47:59 2004 +0000 @@ -0,0 +1,349 @@ +# +# This module is derived from the module described at: +# http://gnosis.cx/publish/programming/charming_python_15.txt +# +# Author: David Mertz (mertz@gnosis.cx) +# Thanks to: Pat Knight (p.knight@ktgroup.co.uk) +# Gregory Popovitch (greg@gpy.com) +# +# The original module was released under this license, and remains under +# it: +# +# This file is released to the public domain. I (dqm) would +# appreciate it if you choose to keep derived works under terms +# that promote freedom, but obviously am giving up any rights +# to compel such. +# +#$Id: indexer_dbm.py,v 1.1 2004-03-19 04:47:59 richard Exp $ +'''This module provides an indexer class, RoundupIndexer, that stores text +indices in a roundup instance. This class makes searching the content of +messages, string properties and text files possible. +''' +__docformat__ = 'restructuredtext' + +import os, shutil, re, mimetypes, marshal, zlib, errno +from roundup.hyperdb import Link, Multilink + +class Indexer: + '''Indexes information from roundup's hyperdb to allow efficient + searching. + + Three structures are created by the indexer:: + + files {identifier: (fileid, wordcount)} + words {word: {fileid: count}} + fileids {fileid: identifier} + + where identifier is (classname, nodeid, propertyname) + ''' + def __init__(self, db_path): + self.indexdb_path = os.path.join(db_path, 'indexes') + self.indexdb = os.path.join(self.indexdb_path, 'index.db') + self.reindex = 0 + self.quiet = 9 + self.changed = 0 + + # see if we need to reindex because of a change in code + version = os.path.join(self.indexdb_path, 'version') + if (not os.path.exists(self.indexdb_path) or + not os.path.exists(version)): + # for now the file itself is a flag + self.force_reindex() + elif os.path.exists(version): + version = open(version).read() + # check the value and reindex if it's not the latest + if version.strip() != '1': + self.force_reindex() + + def force_reindex(self): + '''Force a reindex condition + ''' + if os.path.exists(self.indexdb_path): + shutil.rmtree(self.indexdb_path) + os.makedirs(self.indexdb_path) + os.chmod(self.indexdb_path, 0775) + open(os.path.join(self.indexdb_path, 'version'), 'w').write('1\n') + self.reindex = 1 + self.changed = 1 + + def should_reindex(self): + '''Should we reindex? + ''' + return self.reindex + + def add_text(self, identifier, text, mime_type='text/plain'): + '''Add some text associated with the (classname, nodeid, property) + identifier. + ''' + # make sure the index is loaded + self.load_index() + + # remove old entries for this identifier + if self.files.has_key(identifier): + self.purge_entry(identifier) + + # split into words + words = self.splitter(text, mime_type) + + # Find new file index, and assign it to identifier + # (_TOP uses trick of negative to avoid conflict with file index) + self.files['_TOP'] = (self.files['_TOP'][0]-1, None) + file_index = abs(self.files['_TOP'][0]) + self.files[identifier] = (file_index, len(words)) + self.fileids[file_index] = identifier + + # find the unique words + filedict = {} + for word in words: + if filedict.has_key(word): + filedict[word] = filedict[word]+1 + else: + filedict[word] = 1 + + # now add to the totals + for word in filedict.keys(): + # each word has a dict of {identifier: count} + if self.words.has_key(word): + entry = self.words[word] + else: + # new word + entry = {} + self.words[word] = entry + + # make a reference to the file for this word + entry[file_index] = filedict[word] + + # save needed + self.changed = 1 + + def splitter(self, text, ftype): + '''Split the contents of a text string into a list of 'words' + ''' + if ftype == 'text/plain': + words = self.text_splitter(text) + else: + return [] + return words + + def text_splitter(self, text): + """Split text/plain string into a list of words + """ + # case insensitive + text = str(text).upper() + + # Split the raw text, losing anything longer than 25 characters + # since that'll be gibberish (encoded text or somesuch) or shorter + # than 3 characters since those short words appear all over the + # place + return re.findall(r'\b\w{2,25}\b', text) + + def search(self, search_terms, klass, ignore={}, + dre=re.compile(r'([^\d]+)(\d+)')): + '''Display search results looking for [search, terms] associated + with the hyperdb Class "klass". Ignore hits on {class: property}. + + "dre" is a helper, not an argument. + ''' + # do the index lookup + hits = self.find(search_terms) + if not hits: + return {} + + designator_propname = {} + for nm, propclass in klass.getprops().items(): + if isinstance(propclass, Link) or isinstance(propclass, Multilink): + designator_propname[propclass.classname] = nm + + # build a dictionary of nodes and their associated messages + # and files + nodeids = {} # this is the answer + propspec = {} # used to do the klass.find + for propname in designator_propname.values(): + propspec[propname] = {} # used as a set (value doesn't matter) + for classname, nodeid, property in hits.values(): + # skip this result if we don't care about this class/property + if ignore.has_key((classname, property)): + continue + + # if it's a property on klass, it's easy + if classname == klass.classname: + if not nodeids.has_key(nodeid): + nodeids[nodeid] = {} + continue + + # make sure the class is a linked one, otherwise ignore + if not designator_propname.has_key(classname): + continue + + # it's a linked class - set up to do the klass.find + linkprop = designator_propname[classname] # eg, msg -> messages + propspec[linkprop][nodeid] = 1 + + # retain only the meaningful entries + for propname, idset in propspec.items(): + if not idset: + del propspec[propname] + + # klass.find tells me the klass nodeids the linked nodes relate to + for resid in klass.find(**propspec): + resid = str(resid) + if not nodeids.has_key(id): + nodeids[resid] = {} + node_dict = nodeids[resid] + # now figure out where it came from + for linkprop in propspec.keys(): + for nodeid in klass.get(resid, linkprop): + if propspec[linkprop].has_key(nodeid): + # OK, this node[propname] has a winner + if not node_dict.has_key(linkprop): + node_dict[linkprop] = [nodeid] + else: + node_dict[linkprop].append(nodeid) + return nodeids + + # we override this to ignore not 2 < word < 25 and also to fix a bug - + # the (fail) case. + def find(self, wordlist): + '''Locate files that match ALL the words in wordlist + ''' + if not hasattr(self, 'words'): + self.load_index() + self.load_index(wordlist=wordlist) + entries = {} + hits = None + for word in wordlist: + if not 2 < len(word) < 25: + # word outside the bounds of what we index - ignore + continue + word = word.upper() + entry = self.words.get(word) # For each word, get index + entries[word] = entry # of matching files + if not entry: # Nothing for this one word (fail) + return {} + if hits is None: + hits = {} + for k in entry.keys(): + if not self.fileids.has_key(k): + raise ValueError, 'Index is corrupted: re-generate it' + hits[k] = self.fileids[k] + else: + # Eliminate hits for every non-match + for fileid in hits.keys(): + if not entry.has_key(fileid): + del hits[fileid] + if hits is None: + return {} + return hits + + segments = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ#_-!" + def load_index(self, reload=0, wordlist=None): + # Unless reload is indicated, do not load twice + if self.index_loaded() and not reload: + return 0 + + # Ok, now let's actually load it + db = {'WORDS': {}, 'FILES': {'_TOP':(0,None)}, 'FILEIDS': {}} + + # Identify the relevant word-dictionary segments + if not wordlist: + segments = self.segments + else: + segments = ['-','#'] + for word in wordlist: + segments.append(word[0].upper()) + + # Load the segments + for segment in segments: + try: + f = open(self.indexdb + segment, 'rb') + except IOError, error: + # probably just nonexistent segment index file + if error.errno != errno.ENOENT: raise + else: + pickle_str = zlib.decompress(f.read()) + f.close() + dbslice = marshal.loads(pickle_str) + if dbslice.get('WORDS'): + # if it has some words, add them + for word, entry in dbslice['WORDS'].items(): + db['WORDS'][word] = entry + if dbslice.get('FILES'): + # if it has some files, add them + db['FILES'] = dbslice['FILES'] + if dbslice.get('FILEIDS'): + # if it has fileids, add them + db['FILEIDS'] = dbslice['FILEIDS'] + + self.words = db['WORDS'] + self.files = db['FILES'] + self.fileids = db['FILEIDS'] + self.changed = 0 + + def save_index(self): + # only save if the index is loaded and changed + if not self.index_loaded() or not self.changed: + return + + # brutal space saver... delete all the small segments + for segment in self.segments: + try: + os.remove(self.indexdb + segment) + except OSError, error: + # probably just nonexistent segment index file + if error.errno != errno.ENOENT: raise + + # First write the much simpler filename/fileid dictionaries + dbfil = {'WORDS':None, 'FILES':self.files, 'FILEIDS':self.fileids} + open(self.indexdb+'-','wb').write(zlib.compress(marshal.dumps(dbfil))) + + # The hard part is splitting the word dictionary up, of course + letters = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ#_" + segdicts = {} # Need batch of empty dicts + for segment in letters: + segdicts[segment] = {} + for word, entry in self.words.items(): # Split into segment dicts + initchar = word[0].upper() + segdicts[initchar][word] = entry + + # save + for initchar in letters: + db = {'WORDS':segdicts[initchar], 'FILES':None, 'FILEIDS':None} + pickle_str = marshal.dumps(db) + filename = self.indexdb + initchar + pickle_fh = open(filename, 'wb') + pickle_fh.write(zlib.compress(pickle_str)) + os.chmod(filename, 0664) + + # save done + self.changed = 0 + + def purge_entry(self, identifier): + '''Remove a file from file index and word index + ''' + self.load_index() + + if not self.files.has_key(identifier): + return + + file_index = self.files[identifier][0] + del self.files[identifier] + del self.fileids[file_index] + + # The much harder part, cleanup the word index + for key, occurs in self.words.items(): + if occurs.has_key(file_index): + del occurs[file_index] + + # save needed + self.changed = 1 + + def index_loaded(self): + return (hasattr(self,'fileids') and hasattr(self,'files') and + hasattr(self,'words')) + + + def rollback(self): + ''' load last saved index info. ''' + self.load_index(reload=1) + +# vim: set filetype=python ts=4 sw=4 et si
--- a/roundup/backends/rdbms_common.py Thu Mar 18 22:47:00 2004 +0000 +++ b/roundup/backends/rdbms_common.py Fri Mar 19 04:47:59 2004 +0000 @@ -1,4 +1,4 @@ -# $Id: rdbms_common.py,v 1.81 2004-03-18 01:58:45 richard Exp $ +# $Id: rdbms_common.py,v 1.82 2004-03-19 04:47:59 richard Exp $ ''' Relational database (SQL) backend common code. Basics: @@ -39,7 +39,7 @@ # support from blobfiles import FileStorage -from roundup.indexer import Indexer +from indexer_dbm import Indexer from sessions_rdbms import Sessions, OneTimeKeys from roundup.date import Range @@ -249,7 +249,7 @@ print >>hyperdb.DEBUG, 'update_class FIRING' # detect key prop change for potential index change - keyprop_changes = 0 + keyprop_changes = {} if new_spec[0] != old_spec[0]: keyprop_changes = {'remove': old_spec[0], 'add': new_spec[0]} @@ -260,20 +260,20 @@ if new_has(name): continue - if isinstance(prop, Multilink): + if prop.find('Multilink to') != -1: # first drop indexes. - self.drop_multilink_table_indexes(spec.classname, ml) + self.drop_multilink_table_indexes(spec.classname, name) # now the multilink table itself - sql = 'drop table %s_%s'%(spec.classname, prop) + sql = 'drop table %s_%s'%(spec.classname, name) else: # if this is the key prop, drop the index first if old_spec[0] == prop: - self.drop_class_table_key_index(spec.classname, prop) + self.drop_class_table_key_index(spec.classname, name) del keyprop_changes['remove'] # drop the column - sql = 'alter table _%s drop column _%s'%(spec.classname, prop) + sql = 'alter table _%s drop column _%s'%(spec.classname, name) if __debug__: print >>hyperdb.DEBUG, 'update_class', (self, sql) @@ -974,8 +974,8 @@ ''' Load the journal from the database ''' # now get the journal entries - sql = 'select %s from %s__journal where nodeid=%s'%(cols, classname, - self.arg) + sql = 'select %s from %s__journal where nodeid=%s order by date'%( + cols, classname, self.arg) if __debug__: print >>hyperdb.DEBUG, 'load_journal', (self, sql, nodeid) self.cursor.execute(sql, (nodeid,)) @@ -1019,14 +1019,8 @@ self.sql_commit() # now, do all the other transaction stuff - reindex = {} for method, args in self.transactions: - reindex[method(*args)] = 1 - - # reindex the nodes that request it - for classname, nodeid in filter(None, reindex.keys()): - print >>hyperdb.DEBUG, 'commit.reindex', (classname, nodeid) - self.getclass(classname).index(nodeid) + method(*args) # save the indexer state self.indexer.save_index() @@ -1241,6 +1235,7 @@ elif isinstance(prop, String): if type(value) != type('') and type(value) != type(u''): raise TypeError, 'new property "%s" not a string'%key + self.db.indexer.add_text((self.classname, newid, key), value) elif isinstance(prop, Password): if not isinstance(value, password.Password): @@ -1465,6 +1460,15 @@ If the value of a Link or Multilink property contains an invalid node id, a ValueError is raised. ''' + self.fireAuditors('set', nodeid, propvalues) + oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid)) + propvalues = self.set_inner(nodeid, **propvalues) + self.fireReactors('set', nodeid, oldvalues) + return propvalues + + def set_inner(self, nodeid, **propvalues): + ''' Called by set, in-between the audit and react calls. + ''' if not propvalues: return propvalues @@ -1479,12 +1483,6 @@ if self.db.journaltag is None: raise DatabaseError, 'Database open read-only' - self.fireAuditors('set', nodeid, propvalues) - # Take a copy of the node dict so that the subsequent set - # operation doesn't modify the oldvalues structure. - # XXX used to try the cache here first - oldvalues = copy.deepcopy(self.db.getnode(self.classname, nodeid)) - node = self.db.getnode(self.classname, nodeid) if self.is_retired(nodeid): raise IndexError, 'Requested item is retired' @@ -1620,6 +1618,8 @@ elif isinstance(prop, String): if value is not None and type(value) != type('') and type(value) != type(u''): raise TypeError, 'new property "%s" not a string'%propname + self.db.indexer.add_text((self.classname, nodeid, propname), + value) elif isinstance(prop, Password): if not isinstance(value, password.Password): @@ -1659,8 +1659,6 @@ if self.do_journal: self.db.addjournal(self.classname, nodeid, 'set', journalvalues) - self.fireReactors('set', nodeid, oldvalues) - return propvalues def retire(self, nodeid): @@ -2234,15 +2232,8 @@ # find all the String properties that have indexme for prop, propclass in self.getprops().items(): if isinstance(propclass, String) and propclass.indexme: - try: - value = str(self.get(nodeid, prop)) - except IndexError: - # node no longer exists - entry should be removed - self.db.indexer.purge_entry((self.classname, nodeid, prop)) - else: - # and index them under (classname, nodeid, property) - self.db.indexer.add_text((self.classname, nodeid, prop), - value) + self.db.indexer.add_text((self.classname, nodeid, prop), + str(self.get(nodeid, prop))) # @@ -2297,7 +2288,14 @@ del propvalues['content'] # do the database create - newid = Class.create_inner(self, **propvalues) + newid = self.create_inner(**propvalues) + + # figure the mime type + mime_type = propvalues.get('type', self.default_mime_type) + + # and index! + self.db.indexer.add_text((self.classname, newid, 'content'), content, + mime_type) # fire reactors self.fireReactors('create', newid, None) @@ -2354,27 +2352,34 @@ d['content'] = hyperdb.String() return d - def index(self, nodeid): - ''' Index the node in the search index. + def set(self, itemid, **propvalues): + ''' Snarf the "content" propvalue and update it in a file + ''' + self.fireAuditors('set', itemid, propvalues) + oldvalues = copy.deepcopy(self.db.getnode(self.classname, itemid)) - We want to index the content in addition to the normal String - property indexing. - ''' - # perform normal indexing - Class.index(self, nodeid) + # now remove the content property so it's not stored in the db + content = None + if propvalues.has_key('content'): + content = propvalues['content'] + del propvalues['content'] - # get the content to index - content = self.get(nodeid, 'content') + # do the database create + propvalues = self.set_inner(itemid, **propvalues) - # figure the mime type - if self.properties.has_key('type'): - mime_type = self.get(nodeid, 'type') - else: - mime_type = self.default_mime_type + # do content? + if content: + # store and index + self.db.storefile(self.classname, itemid, None, content) + mime_type = propvalues.get('type', self.get(itemid, 'type')) + if not mime_type: + mime_type = self.default_mime_type + self.db.indexer.add_text((self.classname, itemid, 'content'), + content, mime_type) - # and index! - self.db.indexer.add_text((self.classname, nodeid, 'content'), content, - mime_type) + # fire reactors + self.fireReactors('set', itemid, oldvalues) + return propvalues # XXX deviation from spec - was called ItemClass class IssueClass(Class, roundupdb.IssueClass):
--- a/roundup/backends/sessions_dbm.py Thu Mar 18 22:47:00 2004 +0000 +++ b/roundup/backends/sessions_dbm.py Fri Mar 19 04:47:59 2004 +0000 @@ -1,4 +1,4 @@ -#$Id: sessions_dbm.py,v 1.1 2004-03-18 01:58:45 richard Exp $ +#$Id: sessions_dbm.py,v 1.2 2004-03-19 04:47:59 richard Exp $ """This module defines a very basic store that's used by the CGI interface to store session and one-time-key information. @@ -63,7 +63,9 @@ db = self.opendb('c') try: try: - return marshal.loads(db[infoid]) + d = marshal.loads(db[infoid]) + del d['__timestamp'] + return d except KeyError: raise KeyError, 'No such %s "%s"'%(self.name, infoid) finally:
--- a/roundup/indexer.py Thu Mar 18 22:47:00 2004 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,344 +0,0 @@ -# -# This module is derived from the module described at: -# http://gnosis.cx/publish/programming/charming_python_15.txt -# -# Author: David Mertz (mertz@gnosis.cx) -# Thanks to: Pat Knight (p.knight@ktgroup.co.uk) -# Gregory Popovitch (greg@gpy.com) -# -# The original module was released under this license, and remains under -# it: -# -# This file is released to the public domain. I (dqm) would -# appreciate it if you choose to keep derived works under terms -# that promote freedom, but obviously am giving up any rights -# to compel such. -# -#$Id: indexer.py,v 1.18 2004-02-11 23:55:08 richard Exp $ -'''This module provides an indexer class, RoundupIndexer, that stores text -indices in a roundup instance. This class makes searching the content of -messages, string properties and text files possible. -''' -__docformat__ = 'restructuredtext' - -import os, shutil, re, mimetypes, marshal, zlib, errno -from hyperdb import Link, Multilink - -class Indexer: - '''Indexes information from roundup's hyperdb to allow efficient - searching. - - Three structures are created by the indexer:: - - files {identifier: (fileid, wordcount)} - words {word: {fileid: count}} - fileids {fileid: identifier} - - where identifier is (classname, nodeid, propertyname) - ''' - def __init__(self, db_path): - self.indexdb_path = os.path.join(db_path, 'indexes') - self.indexdb = os.path.join(self.indexdb_path, 'index.db') - self.reindex = 0 - self.quiet = 9 - self.changed = 0 - - # see if we need to reindex because of a change in code - version = os.path.join(self.indexdb_path, 'version') - if (not os.path.exists(self.indexdb_path) or - not os.path.exists(version)): - # for now the file itself is a flag - self.force_reindex() - elif os.path.exists(version): - version = open(version).read() - # check the value and reindex if it's not the latest - if version.strip() != '1': - self.force_reindex() - - def force_reindex(self): - '''Force a reindex condition - ''' - if os.path.exists(self.indexdb_path): - shutil.rmtree(self.indexdb_path) - os.makedirs(self.indexdb_path) - os.chmod(self.indexdb_path, 0775) - open(os.path.join(self.indexdb_path, 'version'), 'w').write('1\n') - self.reindex = 1 - self.changed = 1 - - def should_reindex(self): - '''Should we reindex? - ''' - return self.reindex - - def add_text(self, identifier, text, mime_type='text/plain'): - '''Add some text associated with the (classname, nodeid, property) - identifier. - ''' - # make sure the index is loaded - self.load_index() - - # remove old entries for this identifier - if self.files.has_key(identifier): - self.purge_entry(identifier) - - # split into words - words = self.splitter(text, mime_type) - - # Find new file index, and assign it to identifier - # (_TOP uses trick of negative to avoid conflict with file index) - self.files['_TOP'] = (self.files['_TOP'][0]-1, None) - file_index = abs(self.files['_TOP'][0]) - self.files[identifier] = (file_index, len(words)) - self.fileids[file_index] = identifier - - # find the unique words - filedict = {} - for word in words: - if filedict.has_key(word): - filedict[word] = filedict[word]+1 - else: - filedict[word] = 1 - - # now add to the totals - for word in filedict.keys(): - # each word has a dict of {identifier: count} - if self.words.has_key(word): - entry = self.words[word] - else: - # new word - entry = {} - self.words[word] = entry - - # make a reference to the file for this word - entry[file_index] = filedict[word] - - # save needed - self.changed = 1 - - def splitter(self, text, ftype): - '''Split the contents of a text string into a list of 'words' - ''' - if ftype == 'text/plain': - words = self.text_splitter(text) - else: - return [] - return words - - def text_splitter(self, text): - """Split text/plain string into a list of words - """ - # case insensitive - text = text.upper() - - # Split the raw text, losing anything longer than 25 characters - # since that'll be gibberish (encoded text or somesuch) or shorter - # than 3 characters since those short words appear all over the - # place - return re.findall(r'\b\w{2,25}\b', text) - - def search(self, search_terms, klass, ignore={}, - dre=re.compile(r'([^\d]+)(\d+)')): - '''Display search results looking for [search, terms] associated - with the hyperdb Class "klass". Ignore hits on {class: property}. - - "dre" is a helper, not an argument. - ''' - # do the index lookup - hits = self.find(search_terms) - if not hits: - return {} - - designator_propname = {} - for nm, propclass in klass.getprops().items(): - if isinstance(propclass, Link) or isinstance(propclass, Multilink): - designator_propname[propclass.classname] = nm - - # build a dictionary of nodes and their associated messages - # and files - nodeids = {} # this is the answer - propspec = {} # used to do the klass.find - for propname in designator_propname.values(): - propspec[propname] = {} # used as a set (value doesn't matter) - for classname, nodeid, property in hits.values(): - # skip this result if we don't care about this class/property - if ignore.has_key((classname, property)): - continue - - # if it's a property on klass, it's easy - if classname == klass.classname: - if not nodeids.has_key(nodeid): - nodeids[nodeid] = {} - continue - - # make sure the class is a linked one, otherwise ignore - if not designator_propname.has_key(classname): - continue - - # it's a linked class - set up to do the klass.find - linkprop = designator_propname[classname] # eg, msg -> messages - propspec[linkprop][nodeid] = 1 - - # retain only the meaningful entries - for propname, idset in propspec.items(): - if not idset: - del propspec[propname] - - # klass.find tells me the klass nodeids the linked nodes relate to - for resid in klass.find(**propspec): - resid = str(resid) - if not nodeids.has_key(id): - nodeids[resid] = {} - node_dict = nodeids[resid] - # now figure out where it came from - for linkprop in propspec.keys(): - for nodeid in klass.get(resid, linkprop): - if propspec[linkprop].has_key(nodeid): - # OK, this node[propname] has a winner - if not node_dict.has_key(linkprop): - node_dict[linkprop] = [nodeid] - else: - node_dict[linkprop].append(nodeid) - return nodeids - - # we override this to ignore not 2 < word < 25 and also to fix a bug - - # the (fail) case. - def find(self, wordlist): - '''Locate files that match ALL the words in wordlist - ''' - if not hasattr(self, 'words'): - self.load_index() - self.load_index(wordlist=wordlist) - entries = {} - hits = None - for word in wordlist: - if not 2 < len(word) < 25: - # word outside the bounds of what we index - ignore - continue - word = word.upper() - entry = self.words.get(word) # For each word, get index - entries[word] = entry # of matching files - if not entry: # Nothing for this one word (fail) - return {} - if hits is None: - hits = {} - for k in entry.keys(): - if not self.fileids.has_key(k): - raise ValueError, 'Index is corrupted: re-generate it' - hits[k] = self.fileids[k] - else: - # Eliminate hits for every non-match - for fileid in hits.keys(): - if not entry.has_key(fileid): - del hits[fileid] - if hits is None: - return {} - return hits - - segments = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ#_-!" - def load_index(self, reload=0, wordlist=None): - # Unless reload is indicated, do not load twice - if self.index_loaded() and not reload: - return 0 - - # Ok, now let's actually load it - db = {'WORDS': {}, 'FILES': {'_TOP':(0,None)}, 'FILEIDS': {}} - - # Identify the relevant word-dictionary segments - if not wordlist: - segments = self.segments - else: - segments = ['-','#'] - for word in wordlist: - segments.append(word[0].upper()) - - # Load the segments - for segment in segments: - try: - f = open(self.indexdb + segment, 'rb') - except IOError, error: - # probably just nonexistent segment index file - if error.errno != errno.ENOENT: raise - else: - pickle_str = zlib.decompress(f.read()) - f.close() - dbslice = marshal.loads(pickle_str) - if dbslice.get('WORDS'): - # if it has some words, add them - for word, entry in dbslice['WORDS'].items(): - db['WORDS'][word] = entry - if dbslice.get('FILES'): - # if it has some files, add them - db['FILES'] = dbslice['FILES'] - if dbslice.get('FILEIDS'): - # if it has fileids, add them - db['FILEIDS'] = dbslice['FILEIDS'] - - self.words = db['WORDS'] - self.files = db['FILES'] - self.fileids = db['FILEIDS'] - self.changed = 0 - - def save_index(self): - # only save if the index is loaded and changed - if not self.index_loaded() or not self.changed: - return - - # brutal space saver... delete all the small segments - for segment in self.segments: - try: - os.remove(self.indexdb + segment) - except OSError, error: - # probably just nonexistent segment index file - if error.errno != errno.ENOENT: raise - - # First write the much simpler filename/fileid dictionaries - dbfil = {'WORDS':None, 'FILES':self.files, 'FILEIDS':self.fileids} - open(self.indexdb+'-','wb').write(zlib.compress(marshal.dumps(dbfil))) - - # The hard part is splitting the word dictionary up, of course - letters = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ#_" - segdicts = {} # Need batch of empty dicts - for segment in letters: - segdicts[segment] = {} - for word, entry in self.words.items(): # Split into segment dicts - initchar = word[0].upper() - segdicts[initchar][word] = entry - - # save - for initchar in letters: - db = {'WORDS':segdicts[initchar], 'FILES':None, 'FILEIDS':None} - pickle_str = marshal.dumps(db) - filename = self.indexdb + initchar - pickle_fh = open(filename, 'wb') - pickle_fh.write(zlib.compress(pickle_str)) - os.chmod(filename, 0664) - - # save done - self.changed = 0 - - def purge_entry(self, identifier): - '''Remove a file from file index and word index - ''' - self.load_index() - - if not self.files.has_key(identifier): - return - - file_index = self.files[identifier][0] - del self.files[identifier] - del self.fileids[file_index] - - # The much harder part, cleanup the word index - for key, occurs in self.words.items(): - if occurs.has_key(file_index): - del occurs[file_index] - - # save needed - self.changed = 1 - - def index_loaded(self): - return (hasattr(self,'fileids') and hasattr(self,'files') and - hasattr(self,'words')) - -# vim: set filetype=python ts=4 sw=4 et si
--- a/roundup/roundupdb.py Thu Mar 18 22:47:00 2004 +0000 +++ b/roundup/roundupdb.py Fri Mar 19 04:47:59 2004 +0000 @@ -15,7 +15,7 @@ # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. # -# $Id: roundupdb.py,v 1.101 2004-03-15 05:50:19 richard Exp $ +# $Id: roundupdb.py,v 1.102 2004-03-19 04:47:59 richard Exp $ """Extending hyperdb with types specific to issue-tracking. """ @@ -60,7 +60,7 @@ return timezone def confirm_registration(self, otk): - props = self.otks.getall(otk) + props = self.getOTKManager().getall(otk) for propname, proptype in self.user.getprops().items(): value = props.get(propname, None) if value is None: @@ -80,10 +80,9 @@ cl = self.user props['roles'] = self.config.NEW_WEB_USER_ROLES - del props['__time'] userid = cl.create(**props) # clear the props from the otk database - self.otks.destroy(otk) + self.getOTKManager().destroy(otk) self.commit() return userid
--- a/test/db_test_base.py Thu Mar 18 22:47:00 2004 +0000 +++ b/test/db_test_base.py Fri Mar 19 04:47:59 2004 +0000 @@ -15,7 +15,7 @@ # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. # -# $Id: db_test_base.py,v 1.17 2004-03-18 01:58:46 richard Exp $ +# $Id: db_test_base.py,v 1.18 2004-03-19 04:47:59 richard Exp $ import unittest, os, shutil, errno, imp, sys, time, pprint @@ -23,7 +23,6 @@ Interval, DatabaseError, Boolean, Number, Node from roundup import date, password from roundup import init -from roundup.indexer import Indexer def setupSchema(db, create, module): status = module.Class(db, "status", name=String()) @@ -89,18 +88,20 @@ # automatic properties (well, the two easy ones anyway) # def testCreatorProperty(self): - id1 = self.db.issue.create() + i = self.db.issue + id1 = i.create(title='spam') self.db.commit() self.db.close() self.db = self.module.Database(config, 'fred') setupSchema(self.db, 0, self.module) i = self.db.issue - id2 = i.create() + id2 = i.create(title='spam') self.assertNotEqual(id1, id2) self.assertNotEqual(i.get(id1, 'creator'), i.get(id2, 'creator')) def testActorProperty(self): - id1 = self.db.issue.create() + i = self.db.issue + id1 = i.create(title='spam') self.db.commit() self.db.close() self.db = self.module.Database(config, 'fred') @@ -121,6 +122,7 @@ id1 = self.db.issue.create(title="spam", status='1') self.db.issue.set(id1) + # String def testStringChange(self): for commit in (0,1): # test set & retrieve @@ -142,6 +144,19 @@ if commit: self.db.commit() self.assertEqual(self.db.issue.get(nid, "title"), None) + # FileClass "content" property (no unset test) + def testFileClassContentChange(self): + for commit in (0,1): + # test set & retrieve + nid = self.db.file.create(content="spam") + self.assertEqual(self.db.file.get(nid, 'content'), 'spam') + + # change and make sure we retrieve the correct value + self.db.file.set(nid, content='eggs') + if commit: self.db.commit() + self.assertEqual(self.db.file.get(nid, 'content'), 'eggs') + + # Link def testLinkChange(self): self.assertRaises(IndexError, self.db.issue.create, title="spam", status='100') @@ -161,6 +176,7 @@ if commit: self.db.commit() self.assertEqual(self.db.issue.get(nid, "status"), None) + # Multilink def testMultilinkChange(self): for commit in (0,1): self.assertRaises(IndexError, self.db.issue.create, title="spam", @@ -175,8 +191,11 @@ self.assertEqual(self.db.issue.get(nid, "nosy"), []) self.db.issue.set(nid, nosy=[u1,u2]) if commit: self.db.commit() - self.assertEqual(self.db.issue.get(nid, "nosy"), [u1,u2]) + l = [u1,u2]; l.sort() + m = self.db.issue.get(nid, "nosy"); m.sort() + self.assertEqual(l, m) + # Date def testDateChange(self): self.assertRaises(TypeError, self.db.issue.create, title='spam', deadline=1) @@ -201,6 +220,7 @@ if commit: self.db.commit() self.assertEqual(self.db.issue.get(nid, "deadline"), None) + # Interval def testIntervalChange(self): self.assertRaises(TypeError, self.db.issue.create, title='spam', foo=1) @@ -230,6 +250,7 @@ if commit: self.db.commit() self.assertEqual(self.db.issue.get(nid, "foo"), None) + # Boolean def testBooleanChange(self): userid = self.db.user.create(username='foo', assignable=1) self.assertEqual(1, self.db.user.get(userid, 'assignable')) @@ -241,6 +262,7 @@ self.db.user.set(nid, assignable=None) self.assertEqual(self.db.user.get(nid, "assignable"), None) + # Number def testNumberChange(self): nid = self.db.user.create(username='foo', age=1) self.assertEqual(1, self.db.user.get(nid, 'age')) @@ -259,6 +281,7 @@ self.db.user.set(nid, age=None) self.assertEqual(self.db.user.get(nid, "age"), None) + # Password def testPasswordChange(self): x = password.Password('x') userid = self.db.user.create(username='foo', password=x) @@ -277,6 +300,7 @@ self.db.user.set(nid, assignable=None) self.assertEqual(self.db.user.get(nid, "assignable"), None) + # key value def testKeyValue(self): self.assertRaises(ValueError, self.db.user.create) @@ -295,6 +319,7 @@ self.assertRaises(TypeError, self.db.issue.lookup, 'fubar') + # label property def testLabelProp(self): # key prop self.assertEqual(self.db.status.labelprop(), 'name') @@ -306,6 +331,7 @@ # id self.assertEqual(self.db.stuff.labelprop(default_to_id=1), 'id') + # retirement def testRetire(self): self.db.issue.create(title="spam", status='1') b = self.db.status.get('1', 'name') @@ -609,6 +635,7 @@ def testIndexerSearching(self): f1 = self.db.file.create(content='hello', type="text/plain") + # content='world' has the wrong content-type and won't be indexed f2 = self.db.file.create(content='world', type="text/frozz", comment='blah blah') i1 = self.db.issue.create(files=[f1, f2], title="flebble plop") @@ -623,15 +650,44 @@ {i1: {}, i2: {}}) def testReindexing(self): - self.db.issue.create(title="frooz") + search = self.db.indexer.search + issue = self.db.issue + i1 = issue.create(title="flebble plop") + i2 = issue.create(title="flebble frooz") + self.db.commit() + self.assertEquals(search(['plop'], issue), {i1: {}}) + self.assertEquals(search(['flebble'], issue), {i1: {}, i2: {}}) + + # change i1's title + issue.set(i1, title="plop") + self.db.commit() + self.assertEquals(search(['plop'], issue), {i1: {}}) + self.assertEquals(search(['flebble'], issue), {i2: {}}) + + # unset i1's title + issue.set(i1, title="") self.db.commit() - self.assertEquals(self.db.indexer.search(['frooz'], self.db.issue), - {'1': {}}) - self.db.issue.set('1', title="dooble") + self.assertEquals(search(['plop'], issue), {}) + self.assertEquals(search(['flebble'], issue), {i2: {}}) + + def testFileClassReindexing(self): + f1 = self.db.file.create(content='hello') + f2 = self.db.file.create(content='hello, world') + i1 = self.db.issue.create(files=[f1, f2]) self.db.commit() - self.assertEquals(self.db.indexer.search(['dooble'], self.db.issue), - {'1': {}}) - self.assertEquals(self.db.indexer.search(['frooz'], self.db.issue), {}) + d = self.db.indexer.search(['hello'], self.db.issue) + d[i1]['files'].sort() + self.assertEquals(d, {i1: {'files': [f1, f2]}}) + self.assertEquals(self.db.indexer.search(['world'], self.db.issue), + {i1: {'files': [f2]}}) + self.db.file.set(f1, content="world") + self.db.commit() + d = self.db.indexer.search(['world'], self.db.issue) + d[i1]['files'].sort() + self.assertEquals(d, {i1: {'files': [f1, f2]}}) + self.assertEquals(self.db.indexer.search(['hello'], self.db.issue), + {i1: {'files': [f2]}}) + def testForcedReindexing(self): self.db.issue.create(title="flebble frooz") @@ -889,10 +945,14 @@ ae(l, m) for id, props in items.items(): for name, value in props.items(): - ae(klass.get(id, name), value) + l = klass.get(id, name) + if isinstance(value, type([])): + value.sort() + l.sort() + ae(l, value) # make sure the retired items are actually imported - ae(self.db.user.get('3', 'username'), 'blop') + ae(self.db.user.get('4', 'username'), 'blop') ae(self.db.issue.get('2', 'title'), 'issue two') # make sure id counters are set correctly
--- a/test/session_common.py Thu Mar 18 22:47:00 2004 +0000 +++ b/test/session_common.py Fri Mar 19 04:47:59 2004 +0000 @@ -32,12 +32,6 @@ self.sessions.set('random_key', text='nope') self.assertEqual(self.sessions.get('random_key', 'text'), 'nope') - def testSetOTK(self): - assert 0, 'not implemented' - - def testExpiry(self): - assert 0, 'not implemented' - class DBMTest(SessionTest): import roundup.backends.sessions_dbm as sessions_module
--- a/test/test_indexer.py Thu Mar 18 22:47:00 2004 +0000 +++ b/test/test_indexer.py Fri Mar 19 04:47:59 2004 +0000 @@ -18,11 +18,11 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -# $Id: test_indexer.py,v 1.3 2003-10-25 22:53:26 richard Exp $ +# $Id: test_indexer.py,v 1.4 2004-03-19 04:47:59 richard Exp $ import os, unittest, shutil -from roundup.indexer import Indexer +from roundup.backends.indexer_dbm import Indexer class IndexerTest(unittest.TestCase): def setUp(self):
--- a/test/test_mailgw.py Thu Mar 18 22:47:00 2004 +0000 +++ b/test/test_mailgw.py Fri Mar 19 04:47:59 2004 +0000 @@ -8,7 +8,7 @@ # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. # -# $Id: test_mailgw.py,v 1.64 2004-01-20 00:11:51 richard Exp $ +# $Id: test_mailgw.py,v 1.65 2004-03-19 04:47:59 richard Exp $ import unittest, tempfile, os, shutil, errno, imp, sys, difflib, rfc822 @@ -926,7 +926,7 @@ def testRegistrationConfirmation(self): otk = "Aj4euk4LZSAdwePohj90SME5SpopLETL" - self.db.otks.set(otk, username='johannes', __time='') + self.db.getOTKManager().set(otk, username='johannes') self._handle_mail('''Content-Type: text/plain; charset="iso-8859-1" From: Chef <chef@bork.bork.bork>