Fix async returns

gwicke · gwicke · commit c784c6fe570c · 2015-04-16T19:40:36.000-07:00
diff --git a/lib/dump_restbase.js b/lib/dump_restbase.js
@@ -22,11 +22,13 @@ function dumpWiki(options) {
     var dumpName = dumpDBName(domain);
     var workDB = options.workDir + '/' + dumpName;
     var dumpDB = options.dumpDir + '/' + dumpName + '.xz';
+
     // If a dump exists, uncompress it & use it as a starting point
     var dumpPromise = P.resolve();
     if (fs.existsSync(dumpDB)) {
         dumpPromise = proc.execFileAsync('pixz', ['-d', dumpDB, workDB]);
     }
+
     return dumpPromise
     .then(function() {
         var dumpOptions = {
@@ -41,9 +43,8 @@ function dumpWiki(options) {
     })
     .then(function() {
         console.log('xz compressing');
-        proc.execFileAsync('pixz', ['-2', workDB, dumpDB]);
+        return proc.execFileAsync('pixz', ['-2', workDB, dumpDB]);
     })
-    .catch(console.log)
     .then(function() {
         return fs.unlinkAsync(workDB);
     })
@@ -64,7 +65,10 @@ function dumpAllWikis (options) {
         return P.each(res.body.items, function(domain) {
             console.log(domain);
             options.domain = domain;
-            return dumpWiki(options);
+            return dumpWiki(options)
+            .catch(function(res) {
+                console.log(domain, res);
+            });
         });
     })
     .then(function() {
diff --git a/lib/htmldump.js b/lib/htmldump.js
@@ -23,11 +23,11 @@ var PromiseStream = require('./PromiseStream');
 var maxConcurrency = 50;
 
 function getArticles (options, res) {
-    var next = res.next || '';
-    if (next === 'finished') {
+    if (!res || res.next === 'finished') {
         // nothing more to do.
-        return P.reject('Articles done');
+        return P.resolve(null);
     }
+    var next = res.next || '';
 
     var url = options.apiURL + '?action=query&generator=allpages&gapfilterredir=nonredirects'
         + '&gaplimit=500&prop=revisions&gapnamespace='
@@ -51,7 +51,7 @@ function getArticles (options, res) {
             }
         });
         var next2 = res2['query-continue']
-            && res2['query-continue'].allpages.gapcontinue;
+            && res2['query-continue'].allpages.gapcontinue || 'finished';
         // XXX
         //next = 'finished';
         return {
@@ -113,9 +113,17 @@ function Dumper (articleChunkStream, options) {
     this.options = options;
     this.articles = [];
     this.waiters = [];
+    this.done = false;
 }
 
 Dumper.prototype.processArticles = function (newArticles) {
+    if (newArticles === null) {
+        this.done = true;
+        while(this.waiters.length) {
+            this.waiters.pop().resolve(null);
+        }
+        return;
+    }
     this.articles = newArticles.articles;
     while(this.waiters.length && this.articles.length) {
         this.waiters.pop().resolve(this.articles.shift());
@@ -143,6 +151,9 @@ Dumper.prototype.next = function () {
     var self = this;
     return this.getArticle()
     .then(function(article) {
+        if (article === null) {
+            return null;
+        }
         var title = article[0];
         var oldid = article[1];
         return dumpArticle(self.options, title, oldid)
@@ -164,7 +175,10 @@ function dumpLoop (options) {
     return new P(function(resolve, reject) {
         function loop () {
             return dumpStream.next()
-            .then(function () {
+            .then(function (res) {
+                if (res === null) {
+                    return resolve();
+                }
                 if (i++ === 10000) {
                     i = 0;
                     process.nextTick(loop);
@@ -173,7 +187,11 @@ function dumpLoop (options) {
                 }
             })
             .catch(function(e) {
-                resolve(e);
+                if (e instanceof String) {
+                    resolve(e);
+                } else {
+                    reject(e);
+                }
             });
         }
 
diff --git a/lib/sqlitestore.js b/lib/sqlitestore.js
@@ -69,7 +69,9 @@ SQLiteStore.prototype.saveArticle = function saveArticle (body, title, oldid) {
 };
 
 SQLiteStore.prototype.close = function () {
-    return this.db.closeAsync();
+    var self = this;
+    return P.delay(1000)
+        .then(function() { return self.db.closeAsync(); });
 };
 
 module.exports = function makeSQLiteStore(options) {
