Allow the user to specify the article fetch concurrency

d00rman · d00rman · commit 464bb36d213b · 2017-01-27T11:22:43.000-08:00
diff --git a/bin/dump_wiki b/bin/dump_wiki
@@ -32,6 +32,10 @@ var argParser = require('yargs')
     .options('verbose', {
         default : true
     })
+    .options('c', {
+        alias: 'concurrency',
+        default: 50
+    })
     //.default('apiURL', 'http://en.wikipedia.org/w/api.php')
     //.default('prefix', 'en.wikipedia.org')
     //.default('ns', '0')
diff --git a/lib/htmldump.js b/lib/htmldump.js
@@ -21,9 +21,6 @@ process.on('SIGUSR2', function() {
 var preq = require('preq');
 var PromiseStream = require('./PromiseStream');
 
-// Article dump parallelism
-var maxConcurrency = 50;
-
 function getArticles (options, res) {
     if (!res || res.next === 'finished') {
         // nothing more to do.
@@ -185,7 +182,7 @@ function dumpLoop (options) {
             {next: ''}, 6);
     var dumper = new Dumper(articleChunkStream, options);
     var dumpStream = new PromiseStream(dumper.next.bind(dumper),
-            undefined, 1, maxConcurrency);
+            undefined, 1, options.concurrency);
 
     var i = 0;
     return new P(function(resolve, reject) {
