diff scripts/import_sf.py @ 3544:5cd1c83dea50

Features and fixes. Feature: - trackers may configure custom stop-words for the full-text indexer Fixed: - fixes in scripts/import_sf.py - fix some unicode bugs in roundup-admin import - Xapian indexer wasn't actually being used - fix indexing of message content on roundup-admin import
author Richard Jones <richard@users.sourceforge.net>
date Mon, 06 Feb 2006 21:00:47 +0000
parents 0735c4587852
children 7f1e2d650486
line wrap: on
line diff
--- a/scripts/import_sf.py	Mon Feb 06 02:35:47 2006 +0000
+++ b/scripts/import_sf.py	Mon Feb 06 21:00:47 2006 +0000
@@ -83,7 +83,7 @@
             aid, url = line.strip().split()
             urls[aid] = url
 
-    for aid, fid in Progress('Fetching files', list(to_fetch)):
+    for aid, fid in support.Progress('Fetching files', list(to_fetch)):
         if fid in got: continue
         if not urls.has_key(aid):
             urls[aid] = get_url(aid)
@@ -391,61 +391,6 @@
     f = open('/tmp/imported/%s-journals.csv'%klass.classname, 'w')
     f.close()
 
-class Progress:
-    '''Progress display for console applications.
-
-    See __main__ block at end of file for sample usage.
-    '''
-    def __init__(self, info, sequence):
-        self.info = info
-        self.sequence = iter(sequence)
-        self.total = len(sequence)
-        self.start = self.now = time.time()
-        self.num = 0
-        self.stepsize = self.total / 100 or 1
-        self.steptimes = []
-        self.display()
-
-    def __iter__(self): return self
-
-    def next(self):
-        self.num += 1
-
-        if self.num > self.total:
-            print self.info, 'done', ' '*(75-len(self.info)-6)
-            sys.stdout.flush()
-            return self.sequence.next()
-
-        if self.num % self.stepsize:
-            return self.sequence.next()
-
-        self.display()
-        return self.sequence.next()
-
-    def display(self):
-        # figure how long we've spent - guess how long to go
-        now = time.time()
-        steptime = now - self.now
-        self.steptimes.insert(0, steptime)
-        if len(self.steptimes) > 5:
-            self.steptimes.pop()
-        steptime = sum(self.steptimes) / len(self.steptimes)
-        self.now = now
-        eta = steptime * ((self.total - self.num)/self.stepsize)
-
-        # tell it like it is (or might be)
-        if now - self.start > 3:
-            M = eta / 60
-            H = M / 60
-            M = M % 60
-            S = eta % 60
-            s = '%s %2d%% (ETA %02d:%02d:%02d)'%(self.info,
-                self.num * 100. / self.total, H, M, S)
-        else:
-            s = '%s %2d%%'%(self.info, self.num * 100. / self.total)
-        sys.stdout.write(s + ' '*(75-len(s)) + '\r')
-        sys.stdout.flush()
-
 if __name__ == '__main__':
     if sys.argv[1] == 'import':
         import_xml(*sys.argv[2:])

Roundup Issue Tracker: http://roundup-tracker.org/