Allow users to specify the URL template to use

d00rman · d00rman · commit 962abb463a27 · 2017-04-14T19:05:37.000-05:00
diff --git a/bin/dump_restbase b/bin/dump_restbase
diff --git a/bin/dump_wiki b/bin/dump_wiki
@@ -8,11 +8,14 @@ var makeDump = require('../lib/htmldump');
 var argParser = require('yargs')
     .usage('Create a HTML dump in a subdir\n\n'
             + '\nExample:\n$0 --domain en.wikipedia.org --ns 0 --apiURL http://en.wikipedia.org/w/api.php')
-    .demand(['apiURL', 'domain', 'ns', 'host'])
+    .demand(['apiURL', 'ns', 'host'])
     .options('h', {
         alias: 'help'
     })
-    .alias('domain', 'prefix')
+    .options('domain', {
+        alias: 'prefix',
+        default: ''
+    })
     .options('d', {
         alias : 'saveDir',
         default : ''
@@ -36,10 +39,14 @@ var argParser = require('yargs')
         alias: 'concurrency',
         default: 50
     })
+    .options('u', {
+        alias: 'url',
+        default: '{{host}}/{{domain}}/v1/page/html/{title}/{oldid}'
+    })
     //.default('apiURL', 'http://en.wikipedia.org/w/api.php')
     //.default('prefix', 'en.wikipedia.org')
     //.default('ns', '0')
-    .default('host', 'http://rest.wikimedia.org');
+    //.default('host', 'http://rest.wikimedia.org');
 
 var argv = argParser.argv;
 if (argv.h) {
diff --git a/lib/htmldump.js b/lib/htmldump.js
@@ -3,6 +3,7 @@
 require('core-js/shim');
 
 var P = require('bluebird');
+var Template = require('swagger-router').Template;
 
 var makeFileStore = require('./filestore');
 var makeSQLiteStore = require('./sqlitestore');
@@ -21,6 +22,9 @@ process.on('SIGUSR2', function() {
 var preq = require('preq');
 var PromiseStream = require('./PromiseStream');
 
+// the request template used for fetching each individual article
+var articleReqTpl;
+
 function getArticles (options, res) {
     if (!res || res.next === 'finished') {
         // nothing more to do.
@@ -92,24 +96,12 @@ function dumpArticle (options, title, oldid) {
             if (options.verbose) {
                 console.log('Dumping', title, oldid);
             }
-            var url = options.host + '/' + options.prefix
-                        + '/v1/page/html/' + encodeURIComponent(title) + '/' + oldid;
-            return preq.get({
-                uri: url,
-                headers: {
-                    'user-agent': options.userAgent,
-                    'accept-encoding': 'gzip'
-                },
-                retries: 5,
-                timeout: 60000,
-                // Request a Buffer by default, don't decode to a String. This
-                // saves CPU cycles, but also a lot of memory as large strings are
-                // stored in the old space of the JS heap while Buffers are stored
-                // outside the JS heap.
-                encoding: null
-            })
+            return preq.get(articleReqTpl.expand({
+                request: {
+                    params: Object.assign({title: title, oldid: oldid}, options)
+                }
+            }))
             .then(function(res) {
-                //console.log('done', title);
                 if (options.store) {
                     return options.store.saveArticle(res.body, title, oldid);
                 }
@@ -220,6 +212,23 @@ function makeDump (options) {
         storeSetup = makeSQLiteStore(options);
     }
 
+    // set up the article request template once on start-up
+    articleReqTpl = new Template({
+        method: 'get',
+        uri: options.url,
+        headers: {
+            'user-agent': options.userAgent,
+            'accept-encoding': 'gzip'
+        },
+        retries: 5,
+        timeout: 60000,
+        // Request a Buffer by default, don't decode to a String. This
+        // saves CPU cycles, but also a lot of memory as large strings are
+        // stored in the old space of the JS heap while Buffers are stored
+        // outside the JS heap.
+        encoding: null
+    });
+
     return storeSetup
     .then(function(store) {
         options.store = store;
diff --git a/package.json b/package.json
@@ -23,6 +23,7 @@
     "core-js": "^1.0.0",
     "preq": "~0.3.1",
     "sqlite3": "^3.0.5",
+    "swagger-router": "^0.5.6",
     "yargs": "~1.2.1"
   }
 }

Original file line number	Diff line number	Diff line change
`@@ -23,6 +23,7 @@`
`23`	`23`	`"core-js": "^1.0.0",`
`24`	`24`	`"preq": "~0.3.1",`
`25`	`25`	`"sqlite3": "^3.0.5",`
	`26`	`+ "swagger-router": "^0.5.6",`
`26`	`27`	`"yargs": "~1.2.1"`
`27`	`28`	`}`
`28`	`29`	`}`