Use subprocesses instead of a single process & up parallelism to three

gwicke · gwicke · commit b041f89bbc2b · 2015-04-16T22:05:16.000-07:00
diff --git a/lib/dump_restbase.js b/lib/dump_restbase.js
@@ -31,15 +31,16 @@ function dumpWiki(options) {
 
     return dumpPromise
     .then(function() {
-        var dumpOptions = {
-            dataBase: workDB,
-            apiURL: 'http://' + domain + '/w/api.php',
-            prefix: domain,
-            ns: 0,
-            host: 'http://rest.wikimedia.org',
-            verbose: options.verbose
-        };
-        return makeDump(dumpOptions);
+        var dumpOptions = [
+            __dirname + '/../bin/dump_wiki',
+            '--dataBase', workDB,
+            '--apiURL', 'http://' + domain + '/w/api.php',
+            '--prefix', domain,
+            '--ns', '0',
+            '--host', 'http://rest.wikimedia.org'
+        ];
+        dumpOptions.push(options.verbose ? '--verbose' : '--no-verbose');
+        return proc.execFileAsync('node', dumpOptions);
     })
     .then(function() {
         console.log('xz compressing');
@@ -69,7 +70,7 @@ function dumpAllWikis (options) {
             .catch(function(res) {
                 console.log(domain, res);
             });
-        }, { concurrency: 2 });
+        }, { concurrency: 3 });
     })
     .then(function() {
         console.log('All dumps done.');
