added and adapted scraper from readable-stream

rvagg · rvagg · commit f26b3346d7c3 · 2014-01-17T21:27:37.000+11:00
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1 @@
+node_modules
diff --git a/build/.gitignore b/build/.gitignore
@@ -0,0 +1 @@
+node_modules
diff --git a/build/build.js b/build/build.js
@@ -0,0 +1,94 @@
+#!/usr/bin/env node
+
+const hyperquest  = require('hyperzip')(require('hyperdirect'))
+    , bl          = require('bl')
+    , fs          = require('fs')
+    , path        = require('path')
+    , cheerio     = require('cheerio')
+
+    , files       = require('./files')
+    , testReplace = require('./test-replacements')
+
+    , srcurlpfx   = 'https://raw.github.com/joyent/node/v' + process.argv[2] + '-release/'
+    , libsrcurl   = srcurlpfx + 'lib/'
+    , testsrcurl  = srcurlpfx + 'test/simple/'
+    , testlisturl = 'https://github.com/joyent/node/tree/v' + process.argv[2] + '-release/test/simple'
+    , libourroot  = path.join(__dirname, '../')
+    , testourroot = path.join(__dirname, '../test/simple/')
+
+
+function processFile (url, out, replacements) {
+  hyperquest(url).pipe(bl(function (err, data) {
+    if (err)
+      throw err
+
+    data = data.toString()
+    replacements.forEach(function (replacement) {
+      data = data.replace.apply(data, replacement)
+    })
+
+    fs.writeFile(out, data, 'utf8', function (err) {
+      if (err)
+        throw err
+
+      console.log('Wrote', out)
+    })
+  }))
+}
+
+function processLibFile (file) {
+  var replacements = files[file]
+    , url          = libsrcurl + file
+    , out          = path.join(libourroot, replacements.out || file)
+
+  processFile(url, out, replacements)
+}
+
+
+function processTestFile (file) {
+  var replacements = testReplace.all
+    , url          = testsrcurl + file
+    , out          = path.join(testourroot, file)
+
+  if (testReplace[file])
+    replacements = replacements.concat(testReplace[file])
+
+  processFile(url, out, replacements)
+}
+
+
+if (!/0\.1\d\.\d+/.test(process.argv[2])) {
+  console.log('Usage: build.js <node version>')
+  return process.exit(-1)
+}
+
+
+//--------------------------------------------------------------------
+// Grab & process files in ../lib/
+
+Object.keys(files).forEach(processLibFile)
+
+//--------------------------------------------------------------------
+// Discover, grab and process all test-string-decoder* files on joyent/node
+
+hyperquest(testlisturl).pipe(bl(function (err, data) {
+  if (err)
+    throw err
+
+  var $ = cheerio.load(data.toString())
+
+  $('table.files .js-directory-link').each(function () {
+    var file = $(this).text()
+    if (/^test-string-decoder/.test(file) || file == 'common.js')
+      processTestFile(file)
+  })
+}))
+
+//--------------------------------------------------------------------
+// Grab the joyent/node test/common.js
+
+processFile(
+    testsrcurl + '../common.js'
+  , path.join(testourroot, '../common.js')
+  , testReplace['common.js']
+)
diff --git a/build/files.js b/build/files.js
@@ -0,0 +1,12 @@
+/* This file lists the files to be fetched from the node repo
+ * in the /lib/ directory which will be placed in the ../lib/
+ * directory after having each of the "replacements" in the
+ * array for that file applied to it. The replacements are
+ * simply the arguments to String#replace, so they can be
+ * strings, regexes, functions.
+ */
+
+module.exports['string_decoder.js'] = [
+]
+
+module.exports['string_decoder.js'].out = 'index.js'
diff --git a/build/package.json b/build/package.json
@@ -0,0 +1,12 @@
+{
+  "name": "string_decoder-build",
+  "version": "0.0.0",
+  "description": "",
+  "main": "build.js",
+  "dependencies": {
+    "bl": "~0.6.0",
+    "hyperzip": "0.0.0",
+    "hyperdirect": "0.0.0",
+    "cheerio": "~0.13.1"
+  }
+}
diff --git a/build/test-replacements.js b/build/test-replacements.js
@@ -0,0 +1,24 @@
+module.exports.all = [
+    [
+        /require\(['"]string_decoder['"]\)/g
+      , 'require(\'../../\')'
+    ]
+
+]
+
+module.exports['common.js'] = [
+    [
+        /^                      setImmediate,$/m
+      , '                      typeof setImmediate == \'undefined\' ? null : setImmediate,'
+    ]
+
+  , [
+        /^                      clearImmediate,$/m
+      , '                      typeof clearImmediate == \'undefined\' ? null : clearImmediate,'
+    ]
+
+  , [
+        /^                      global];$/m
+      , '                      global].filter(Boolean);'
+    ]
+]
