Tune DB with pragmas & perform async setup in .setup() method

gwicke · gwicke · commit 2934d96aa9c5 · 2015-04-07T13:37:40.000-07:00
diff --git a/filestore.js b/filestore.js
@@ -1,11 +1,15 @@
 "use strict";
-var Bluebird = require('bluebird');
-var fs = Bluebird.promisifyAll(require('fs'));
+var P = require('bluebird');
+var fs = P.promisifyAll(require('fs'));
 
 function FileStore(options) {
     this.options = options;
 }
 
+FileStore.prototype.setup = function() {
+    return P.resolve(this);
+};
+
 FileStore.prototype.checkArticle = function checkArticle (title, oldid) {
     var options = this.options;
     var dumpDir = options.saveDir + '/' + options.prefix;
@@ -21,7 +25,6 @@ FileStore.prototype.checkArticle = function checkArticle (title, oldid) {
             // We already have the article, nothing to do.
             // XXX: Also track / check last-modified time for template
             // re-expansions without revisions change
-            console.log('Exists:', title, oldid);
             return true;
         } else {
             return false;
@@ -54,4 +57,6 @@ FileStore.prototype.saveArticle = function saveArticle (body, title, oldid) {
     });
 };
 
-module.exports = FileStore;
+module.exports = function makeFileStore(options) {
+    return new FileStore(options).setup();
+};
diff --git a/htmldumper.js b/htmldumper.js
@@ -1,9 +1,9 @@
 "use strict";
 
-var Bluebird = require('bluebird');
+var P = require('bluebird');
 
-var FileStore = require('./filestore');
-var SQLiteStore = require('./sqlitestore');
+var makeFileStore = require('./filestore');
+var makeSQLiteStore = require('./sqlitestore');
 
 // Enable heap dumps in /tmp on kill -USR2.
 // See https://github.com/bnoordhuis/node-heapdump/
@@ -26,7 +26,7 @@ function getArticles (options, res) {
     var next = res.next || '';
     if (next === 'finished') {
         // nothing more to do.
-        return Bluebird.reject('Articles done');
+        return P.reject('Articles done');
     }
 
     var url = options.apiURL + '?action=query&generator=allpages&gapfilterredir=nonredirects'
@@ -70,7 +70,7 @@ function dumpArticle (options, title, oldid) {
     if (options.store) {
         checkRevision = options.store.checkArticle(title, oldid);
     } else {
-        checkRevision = Bluebird.resolve(false);
+        checkRevision = P.resolve(false);
     }
 
     return checkRevision
@@ -125,12 +125,12 @@ Dumper.prototype.processArticles = function (newArticles) {
 Dumper.prototype.getArticle = function () {
     var self = this;
     if (this.articles.length) {
-        return Bluebird.resolve(this.articles.shift());
+        return P.resolve(this.articles.shift());
     } else {
         if (!this.waiters.length) {
             this.articleChunkStream.next().then(this.processArticles.bind(this));
         }
-        return new Bluebird(function(resolve, reject) {
+        return new P(function(resolve, reject) {
             self.waiters.push({resolve: resolve, reject: reject});
         });
     }
@@ -210,13 +210,18 @@ if (module.parent === null) {
 
     argv.ns = Number(argv.ns);
 
+    var storeSetup = P.resolve();
     if (argv.saveDir) {
-        argv.store = new FileStore(argv);
+        storeSetup = makeFileStore(argv);
     } else if (argv.dataBase) {
-        argv.store = new SQLiteStore(argv);
+        storeSetup = makeSQLiteStore(argv);
     }
 
-    return makeDump(argv)
+    return storeSetup
+    .then(function(store) {
+        argv.store = store;
+        return makeDump(argv);
+    })
     .then(function(res) {
         console.log('Dump done.');
     })
diff --git a/sqlitestore.js b/sqlitestore.js
@@ -3,6 +3,14 @@ var P = require('bluebird');
 var fs = P.promisifyAll(require('fs'));
 var sqlite3 = P.promisifyAll(require('sqlite3'));
 
+var pragmas = [
+    'PRAGMA main.page_size = 4096',
+    'PRAGMA main.cache_size=10000',
+    'PRAGMA main.locking_mode=EXCLUSIVE',
+    'PRAGMA main.synchronous=NORMAL',
+    'PRAGMA main.journal_mode=WAL',
+    'PRAGMA main.cache_size=5000'
+];
 var createTableQuery = 'CREATE TABLE IF NOT EXISTS data('
         + 'title TEXT, revision INTEGER, body BLOB, namespace INTEGER'
         + ', PRIMARY KEY(title ASC, revision DESC)'
@@ -14,16 +22,28 @@ var saveQuery = 'insert into data (title, revision, body, namespace) values (?,?
 function SQLiteStore(options) {
     this.options = options;
     this.db = new sqlite3.Database(options.dataBase);
-    this.db.exec(createTableQuery);
-    this.queries = {
-        check: this.db.prepare(checkQuery),
-        purgeTitle: this.db.prepare(purgeTitleQuery),
-        save: this.db.prepare(saveQuery),
-    };
 }
 
+SQLiteStore.prototype.setup = function() {
+    var self = this;
+    return this.db.execAsync(createTableQuery)
+        .then(function() {
+            return P.all(pragmas.map(function(pragma) {
+                return self.db.execAsync(pragma);
+            }));
+        })
+        .then(function() {
+            self.queries = {
+                check: self.db.prepare(checkQuery),
+                purgeTitle: self.db.prepare(purgeTitleQuery),
+                save: self.db.prepare(saveQuery),
+            };
+            return self;
+        });
+};
+
 SQLiteStore.prototype.checkArticle = function checkArticle (title, oldid) {
-    return this.queries.check.getAsync(title, oldid)
+    return this.queries.check.getAsync(title, oldid);
 };
 
 SQLiteStore.prototype.saveArticle = function saveArticle (body, title, oldid) {
@@ -34,4 +54,6 @@ SQLiteStore.prototype.saveArticle = function saveArticle (body, title, oldid) {
     });
 };
 
-module.exports = SQLiteStore;
+module.exports = function makeSQLiteStore(options) {
+    return new SQLiteStore(options).setup();
+};
