Merge pull request #9 from d00rman/url-template

Pchelolo · web-flow · commit 3ac4a2407884 · 2017-04-18T12:47:35.000-04:00
Allow users to specify the URL template to use for fetching articles
diff --git a/README.md b/README.md
@@ -16,15 +16,39 @@ Example:
 
 Options:
   --apiURL            [required]
-  --domain, --prefix  [required]
   --ns                [required]
-  --host              [required]  [default: "http://rest.wikimedia.org"]
+  --host              [required]
+  --domain, --prefix  [default: ""]
   -d, --saveDir       [default: ""]
   -t, --startTitle    [default: ""]
+  -a, --userAgent     [default: "HTMLDumper"]
   --db, --dataBase    [default: ""]
   --verbose           [default: true]
+  -c, --concurrency   [default: 50]
+  -u, --url           [default: "{{host}}/{{domain}}/v1/page/html/{title}/{oldid}"]
 ```
 
+Parameters:
+- **`apiURL`**: The location of the Wiki's MW Action API end point.
+- **`ns`**: The namespace index to dump.
+- **`host`**: The host to send the dump requests to.
+- **`domain`**: If the host contains multiple domains, the one to reach.
+- **`saveDir`**: If saving the contents of the dump to a directory structure,
+  this is the path to the root of the directory (see the following section).
+- **`startTitle`**: If resuming a Wiki dump, the article title to start with.
+- **`userAgent`**: The UserAgent header to use when sending requests. Default:
+  `HMTLDumper`
+- **`dataBase`**: If saving the contents to a SQLite3 database, the path to the
+  file to save it to (see the next sections).
+- **`verbose`**: Be verbose.
+- **`concurrency`**: The number of parallel article fetches to do. Default:
+  `50`.
+- **`url`**: The [URL
+  template](https://github.com/wikimedia/swagger-router#uri-templating) to use
+  when making requests for each article. The available parameters are: `title`,
+  `oldid` and all of the options that can be set on the command line (`host`,
+  `domain`, etc.). Default: `{{host}}/{{domain}}/v1/page/html/{title}/{oldid}`
+
 ### Filesystem output
 
 With `--saveDir` as specified in the example above, a directory structure like
diff --git a/bin/dump_restbase b/bin/dump_restbase
diff --git a/bin/dump_wiki b/bin/dump_wiki
@@ -8,11 +8,14 @@ var makeDump = require('../lib/htmldump');
 var argParser = require('yargs')
     .usage('Create a HTML dump in a subdir\n\n'
             + '\nExample:\n$0 --domain en.wikipedia.org --ns 0 --apiURL http://en.wikipedia.org/w/api.php')
-    .demand(['apiURL', 'domain', 'ns', 'host'])
+    .demand(['apiURL', 'ns', 'host'])
     .options('h', {
         alias: 'help'
     })
-    .alias('domain', 'prefix')
+    .options('domain', {
+        alias: 'prefix',
+        default: ''
+    })
     .options('d', {
         alias : 'saveDir',
         default : ''
@@ -36,10 +39,14 @@ var argParser = require('yargs')
         alias: 'concurrency',
         default: 50
     })
+    .options('u', {
+        alias: 'url',
+        default: '{{host}}/{{domain}}/v1/page/html/{title}/{oldid}'
+    })
     //.default('apiURL', 'http://en.wikipedia.org/w/api.php')
     //.default('prefix', 'en.wikipedia.org')
     //.default('ns', '0')
-    .default('host', 'http://rest.wikimedia.org');
+    //.default('host', 'http://rest.wikimedia.org');
 
 var argv = argParser.argv;
 if (argv.h) {
diff --git a/lib/htmldump.js b/lib/htmldump.js
@@ -3,6 +3,7 @@
 require('core-js/shim');
 
 var P = require('bluebird');
+var Template = require('swagger-router').Template;
 
 var makeFileStore = require('./filestore');
 var makeSQLiteStore = require('./sqlitestore');
@@ -21,6 +22,9 @@ process.on('SIGUSR2', function() {
 var preq = require('preq');
 var PromiseStream = require('./PromiseStream');
 
+// the request template used for fetching each individual article
+var articleReqTpl;
+
 function getArticles (options, res) {
     if (!res || res.next === 'finished') {
         // nothing more to do.
@@ -92,24 +96,12 @@ function dumpArticle (options, title, oldid) {
             if (options.verbose) {
                 console.log('Dumping', title, oldid);
             }
-            var url = options.host + '/' + options.prefix
-                        + '/v1/page/html/' + encodeURIComponent(title) + '/' + oldid;
-            return preq.get({
-                uri: url,
-                headers: {
-                    'user-agent': options.userAgent,
-                    'accept-encoding': 'gzip'
-                },
-                retries: 5,
-                timeout: 60000,
-                // Request a Buffer by default, don't decode to a String. This
-                // saves CPU cycles, but also a lot of memory as large strings are
-                // stored in the old space of the JS heap while Buffers are stored
-                // outside the JS heap.
-                encoding: null
-            })
+            return preq.get(articleReqTpl.expand({
+                request: {
+                    params: Object.assign({title: title, oldid: oldid}, options)
+                }
+            }))
             .then(function(res) {
-                //console.log('done', title);
                 if (options.store) {
                     return options.store.saveArticle(res.body, title, oldid);
                 }
@@ -220,6 +212,23 @@ function makeDump (options) {
         storeSetup = makeSQLiteStore(options);
     }
 
+    // set up the article request template once on start-up
+    articleReqTpl = new Template({
+        method: 'get',
+        uri: options.url,
+        headers: {
+            'user-agent': options.userAgent,
+            'accept-encoding': 'gzip'
+        },
+        retries: 5,
+        timeout: 60000,
+        // Request a Buffer by default, don't decode to a String. This
+        // saves CPU cycles, but also a lot of memory as large strings are
+        // stored in the old space of the JS heap while Buffers are stored
+        // outside the JS heap.
+        encoding: null
+    });
+
     return storeSetup
     .then(function(store) {
         options.store = store;
diff --git a/package.json b/package.json
@@ -23,6 +23,7 @@
     "core-js": "^1.0.0",
     "preq": "~0.3.1",
     "sqlite3": "^3.0.5",
+    "swagger-router": "^0.5.6",
     "yargs": "~1.2.1"
   }
 }

Original file line number	Diff line number	Diff line change
`@@ -23,6 +23,7 @@`
`23`	`23`	`"core-js": "^1.0.0",`
`24`	`24`	`"preq": "~0.3.1",`
`25`	`25`	`"sqlite3": "^3.0.5",`
	`26`	`+ "swagger-router": "^0.5.6",`
`26`	`27`	`"yargs": "~1.2.1"`
`27`	`28`	`}`
`28`	`29`	`}`