Fix memory leak in promise loop by breaking the recursion

gwicke · gwicke · commit a94d2daed459 · 2014-12-30T00:48:21.000-08:00
It turns out that neither bluebird nor the native Promise implementation in v8
actually perform TCO when a promise is returned. This built up a long chain of
promise references in memory while dumping millions of wikipedia articles. The
dump would suceed, but for enwiki at least at the end the dumper would use &gt;1G
of heap.

Since nextTick and its siblings setInterval and setTimeout are all pretty
expensive, we only break the chain occasionally. In local tests this
construct runs 70000000 iterations in constant memory, while the same loop
would exhaust the default 2G heap limit.
diff --git a/htmlspider.js b/htmlspider.js
@@ -146,9 +146,17 @@ function makeDump (apiURL, prefix, ns, host) {
     var dumpStream = new PromiseStream(dumper.next.bind(dumper),
             undefined, 1, maxConcurrency);
 
+    var i = 0;
     function loop () {
         return dumpStream.next()
-        .then(loop)
+        .then(function () {
+            if (i++ === 10000) {
+                i = 0;
+                process.nextTick(loop);
+            } else {
+                return loop();
+            }
+        })
         .catch(function(e) {
             console.log(e);
         });
