Update for breaking MediaWiki API changes

gwicke · gwicke · commit bd7022285169 · 2015-07-28T18:34:50.000-07:00
- switch continuation behavior to the new-style MediaWiki API
- clean up query string building in the process
- pull in core-js for es6 compatibility

Change-Id: I58fb25a7ac64b3c065c11ad59fca5654513b8c39
diff --git a/lib/htmldump.js b/lib/htmldump.js
@@ -1,4 +1,6 @@
 "use strict";
+// Upgrade to es6
+require('core-js/shim');
 
 var P = require('bluebird');
 
@@ -29,13 +31,22 @@ function getArticles (options, res) {
     }
     var next = res.next || '';
 
-    var url = options.apiURL + '?action=query&generator=allpages'
-        + '&gaplimit=500&prop=revisions&gapnamespace='
-        + options.ns + '&format=json&rawcontinue=1&gapcontinue=' + encodeURIComponent( next );
-    //console.log(url);
-
+    var query = {
+        action: 'query',
+        generator: 'allpages',
+        gaplimit: '500',
+        prop: 'revisions',
+        gapnamespace: options.ns,
+        format: 'json',
+        'continue': '',
+    }
+    // merge in the paging parameters
+    if (next) {
+        Object.assign(query, next);
+    }
     return preq.get({
-        uri: url,
+        uri: options.apiURL,
+        query: query,
         timeout: 60* 1000,
         retries: 5
     })
@@ -50,8 +61,7 @@ function getArticles (options, res) {
                 articles.push([title, article.revisions[0].revid]);
             }
         });
-        var next2 = res2['query-continue']
-            && res2['query-continue'].allpages.gapcontinue || 'finished';
+        var next2 = res2['continue'] || 'finished';
         // XXX
         //next = 'finished';
         return {
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "htmldump",
-  "version": "0.1.0",
+  "version": "0.1.1",
   "description": "Parsoid HTML dump utility",
   "scripts": {
     "test": "echo \"Error: no test specified\" && exit 1"

Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,6 @@`
`1`	`1`	`{`
`2`	`2`	`"name": "htmldump",`
`3`		`- "version": "0.1.0",`
	`3`	`+ "version": "0.1.1",`
`4`	`4`	`"description": "Parsoid HTML dump utility",`
`5`	`5`	`"scripts": {`
`6`	`6`	`"test": "echo \"Error: no test specified\" && exit 1"`