node v7.7.2 build (#262)

* replace bind()

* re-add WriteReq and CorkedRequest constructors

* Updated to v7.7.2, refactored the build to not crawl

The new build system uses the tarball from nodejs.org.

* try updating zuul

* just remove android

Author: mcollina

.travis.yml

@@ -28,8 +28,6 @@ matrix:
     env: TASK=test
   - node_js: 6
     env: TASK=test
-  - node_js: 5
-    env: TASK=browser BROWSER_NAME=android BROWSER_VERSION="4.0..latest"
   - node_js: 5
     env: TASK=browser BROWSER_NAME=ie BROWSER_VERSION="9..latest"
   - node_js: 5

README.md

@@ -18,7 +18,7 @@ npm install --save readable-stream
 This package is a mirror of the Streams2 and Streams3 implementations in
 Node-core.
 
-Full documentation may be found on the [Node.js website](https://nodejs.org/dist/v7.3.0/docs/api/).
+Full documentation may be found on the [Node.js website](https://nodejs.org/dist/v7.7.2/docs/api/).
 
 If you want to guarantee a stable streams base, regardless of what version of
 Node you, or the users of your libraries are using, use **readable-stream** *only* and avoid the *"stream"* module in Node-core, for background see [this blogpost](http://r.va.gg/2014/06/why-i-dont-use-nodes-core-stream-module.html).

build/build.js

@@ -1,13 +1,17 @@
 #!/usr/bin/env node
 
-const hyperquest  = require('hyperzip')(require('hyperdirect'))
+const hyperquest  = require('hyperquest')
     , bl          = require('bl')
     , fs          = require('fs')
     , path        = require('path')
-    , cheerio     = require('cheerio')
+    , tar         = require('tar-fs')
+    , gunzip      = require('gunzip-maybe')
     , babel       = require('babel-core')
+    , glob        = require('glob')
+    , pump        = require('pump')
+    , rimraf      = require('rimraf')
     , encoding    = 'utf8'
-    , urlRegex = /^https?:\/\//
+    , urlRegex    = /^https?:\/\//
     , nodeVersion = process.argv[2]
     , nodeVersionRegexString = '\\d+\\.\\d+\\.\\d+'
     , usageVersionRegex = RegExp('^' + nodeVersionRegexString + '$')
@@ -18,10 +22,10 @@ const hyperquest  = require('hyperzip')(require('hyperdirect'))
     , files       = require('./files')
     , testReplace = require('./test-replacements')
 
-    , srcurlpfx   = `https://raw.githubusercontent.com/nodejs/node/v${nodeVersion}/`
-    , libsrcurl   = srcurlpfx + 'lib/'
-    , testsrcurl  = srcurlpfx + 'test/parallel/'
-    , testlisturl = `https://github.com/nodejs/node/tree/v${nodeVersion}/test/parallel`
+    , downloadurl = `https://nodejs.org/dist/v${nodeVersion}/node-v${nodeVersion}.tar.gz`
+    , src         = path.join(__dirname, `node-v${nodeVersion}`)
+    , libsrcurl   = path.join(src, 'lib/')
+    , testsrcurl  = path.join(src, 'test/parallel/')
     , libourroot  = path.join(__dirname, '../lib/')
     , testourroot = path.join(__dirname, '../test/parallel/')
 
@@ -33,16 +37,19 @@ if (!usageVersionRegex.test(nodeVersion)) {
 
 // `inputLoc`: URL or local path.
 function processFile (inputLoc, out, replacements) {
-  var file = urlRegex.test(inputLoc) ?
-    hyperquest(inputLoc) :
-    fs.createReadStream(inputLoc, encoding)
+  var file = fs.createReadStream(inputLoc, encoding)
 
   file.pipe(bl(function (err, data) {
     if (err) throw err
 
+    console.log('Processing', inputLoc)
     data = data.toString()
     replacements.forEach(function (replacement) {
-      data = data.replace.apply(data, replacement)
+      const regexp = replacement[0]
+      var arg2 = replacement[1]
+      if (typeof arg2 === 'function')
+        arg2 = arg2.bind(data)
+      data = data.replace(regexp, arg2)
     })
     if (inputLoc.slice(-3) === '.js') {
       const transformed = babel.transform(data, {
@@ -69,7 +76,7 @@ function deleteOldTests(){
   const files = fs.readdirSync(path.join(__dirname, '..', 'test', 'parallel'));
   for (let file of files) {
     let name = path.join(__dirname, '..', 'test', 'parallel', file);
-    console.log('removing', name);
+    console.log('Removing', name);
     fs.unlinkSync(name);
   }
 }
@@ -94,43 +101,73 @@ function processTestFile (file) {
 }
 
 //--------------------------------------------------------------------
-// Grab & process files in ../lib/
+// Download the release from nodejs.org
+console.log(`Downloading ${downloadurl}`)
+pump(
+  hyperquest(downloadurl),
+  gunzip(),
+  tar.extract(__dirname),
+  function (err) {
+    if (err) {
+      throw err
+    }
 
-Object.keys(files).forEach(processLibFile)
 
-// delete the current contents of test/parallel so if node removes any tests
-// they are removed here
-deleteOldTests();
+    //--------------------------------------------------------------------
+    // Grab & process files in ../lib/
 
-//--------------------------------------------------------------------
-// Discover, grab and process all test-stream* files on nodejs/node
+    Object.keys(files).forEach(processLibFile)
 
-hyperquest(testlisturl).pipe(bl(function (err, data) {
-  if (err)
-    throw err
 
-  var $ = cheerio.load(data.toString())
+    //--------------------------------------------------------------------
+    // Discover, grab and process all test-stream* files on the given release
 
-  $('table.files .js-navigation-open').each(function () {
-    var file = $(this).text()
-    if (/^test-stream/.test(file) && !/-wrap(?:-encoding)?\.js$/.test(file) && file !== 'test-stream2-httpclient-response-end.js' && file !== 'test-stream-base-no-abort.js' && file !== 'test-stream-preprocess.js' && file !== 'test-stream-inheritance.js')
-      processTestFile(file)
-  })
-}))
+    glob(path.join(testsrcurl, 'test-stream*.js'), function (err, list) {
+      if (err) {
+        throw err
+      }
 
+      list.forEach(function (file) {
+        file = path.basename(file)
+        if (!/-wrap(?:-encoding)?\.js$/.test(file) &&
+            file !== 'test-stream2-httpclient-response-end.js' &&
+            file !== 'test-stream-base-no-abort.js' &&
+            file !== 'test-stream-preprocess.js' &&
+            file !== 'test-stream-inheritance.js') {
+          processTestFile(file)
+        }
+      })
+    })
 
-//--------------------------------------------------------------------
-// Grab the nodejs/node test/common.js
 
-processFile(
-    testsrcurl.replace(/parallel\/$/, 'common.js')
-  , path.join(testourroot, '../common.js')
-  , testReplace['common.js']
+    //--------------------------------------------------------------------
+    // Grab the nodejs/node test/common.js
+
+    processFile(
+        testsrcurl.replace(/parallel\/$/, 'common.js')
+      , path.join(testourroot, '../common.js')
+      , testReplace['common.js']
+    )
+
+    //--------------------------------------------------------------------
+    // Update Node version in README
+
+    processFile(readmePath, readmePath, [
+      [readmeVersionRegex, "$1" + nodeVersion]
+    ])
+  }
 )
 
-//--------------------------------------------------------------------
-// Update Node version in README
+// delete the current contents of test/parallel so if node removes any tests
+// they are removed here
+deleteOldTests();
 
-processFile(readmePath, readmePath, [
-  [readmeVersionRegex, "$1" + nodeVersion]
-])
+process.once('beforeExit', function () {
+  rimraf(src, function (err) {
+    if (err) {
+      throw err
+    }
+
+    console.log('Removed', src)
+  })
+})

build/files.js

@@ -183,6 +183,47 @@ const headRegexp = /(^module.exports = \w+;?)/m
     /if \(typeof Symbol === 'function' && Symbol\.hasInstance\) \{/,
     `if (typeof Symbol === 'function' && Symbol.hasInstance && typeof Function.prototype[Symbol.hasInstance] === 'function') {`
   ]
+  , removeOnWriteBind = [
+    /onwrite\.bind\([^)]+?\)/,
+    `function(er) { onwrite(stream, er); }`
+  ]
+  , removeCorkedFinishBind = [
+    /onCorkedFinish\.bind\([^)]+?\)/,
+    function (match) {
+      const code = this
+      var src = /^function onCorkedFinish[^{]*?\{([\s\S]+?\r?\n)\}/m.exec(code)
+      src = src[1].trim().replace(/corkReq/g, 'this').replace(/(\r?\n)/mg, '  $1')
+      return `(err) => {\n${src}\n}`
+    }
+  ]
+  , removeOnCorkedFinish = [
+    /^function onCorkedFinish[\s\S]+?\r?\n\}/m,
+    ''
+  ]
+  , addConstructors = [
+    headRegexp,
+    `$1\n\nfunction WriteReq(chunk, encoding, cb) {
+      this.chunk = chunk;
+      this.encoding = encoding;
+      this.callback = cb;
+      this.next = null;
+    }
+    // It seems a linked list but it is not
+    // there will be only 2 of these for each stream
+    function CorkedRequest(state) {
+      this.next = null;
+      this.entry = null;
+      this.finish = onCorkedFinish.bind(undefined, this, state);
+    }\n\n`
+  ]
+  , useWriteReq = [
+    /state\.lastBufferedRequest = \{.+?\}/g,
+    `state.lastBufferedRequest = new WriteReq(chunk, encoding, cb)`
+  ]
+  , useCorkedRequest = [
+    /var corkReq = [\s\S]+?(.+?)\.corkedRequestsFree = corkReq/g,
+    `$1.corkedRequestsFree = new CorkedRequest($1)`
+  ]
 
 module.exports['_stream_duplex.js'] = [
     requireReplacement
@@ -261,6 +302,12 @@ module.exports['_stream_writable.js'] = [
   , bufferShimFix
   , bufferStaticMethods
   , fixInstanceCheck
+  , removeOnWriteBind
+  , removeCorkedFinishBind
+  , removeOnCorkedFinish
+  , addConstructors
+  , useWriteReq
+  , useCorkedRequest
 ]
 module.exports['internal/streams/BufferList.js'] = [
     bufferShimFix

build/package.json

@@ -12,9 +12,12 @@
     "babel-plugin-transform-es2015-parameters": "^6.11.4",
     "babel-plugin-transform-es2015-shorthand-properties": "^6.8.0",
     "babel-plugin-transform-es2015-template-literals": "^6.8.0",
-    "bl": "~0.6.0",
-    "cheerio": "~0.13.1",
-    "hyperdirect": "0.0.0",
-    "hyperzip": "0.0.0"
+    "bl": "^1.2.0",
+    "glob": "^7.1.1",
+    "gunzip-maybe": "^1.4.0",
+    "hyperquest": "^2.1.2",
+    "pump": "^1.0.2",
+    "rimraf": "^2.6.1",
+    "tar-fs": "^1.15.1"
   }
 }

lib/_stream_readable.js

@@ -92,7 +92,7 @@ function ReadableState(options, stream) {
   this.highWaterMark = hwm || hwm === 0 ? hwm : defaultHwm;
 
   // cast to ints.
-  this.highWaterMark = ~ ~this.highWaterMark;
+  this.highWaterMark = ~~this.highWaterMark;
 
   // A linked list is used to store data chunks instead of an array because the
   // linked list can remove elements from the beginning faster than

lib/_stream_writable.js

@@ -6,6 +6,20 @@
 
 module.exports = Writable;
 
+function WriteReq(chunk, encoding, cb) {
+  this.chunk = chunk;
+  this.encoding = encoding;
+  this.callback = cb;
+  this.next = null;
+}
+// It seems a linked list but it is not
+// there will be only 2 of these for each stream
+function CorkedRequest(state) {
+  this.next = null;
+  this.entry = null;
+  this.finish = onCorkedFinish.bind(undefined, this, state);
+}
+
 /*<replacement>*/
 var processNextTick = require('process-nextick-args');
 /*</replacement>*/
@@ -77,7 +91,7 @@ function WritableState(options, stream) {
   this.highWaterMark = hwm || hwm === 0 ? hwm : defaultHwm;
 
   // cast to ints.
-  this.highWaterMark = ~ ~this.highWaterMark;
+  this.highWaterMark = ~~this.highWaterMark;
 
   // drain event flag.
   this.needDrain = false;
@@ -232,20 +246,16 @@ function writeAfterEnd(stream, cb) {
   processNextTick(cb, er);
 }
 
-// If we get something that is not a buffer, string, null, or undefined,
-// and we're not in objectMode, then that's an error.
-// Otherwise stream chunks are all considered to be of length=1, and the
-// watermarks determine how many objects to keep in the buffer, rather than
-// how many bytes or characters.
+// Checks that a user-supplied chunk is valid, especially for the particular
+// mode the stream is in. Currently this means that `null` is never accepted
+// and undefined/non-string values are only allowed in object mode.
 function validChunk(stream, state, chunk, cb) {
   var valid = true;
   var er = false;
-  // Always throw error if a null is written
-  // if we are not in object mode then throw
-  // if it is not a buffer, string, or undefined.
+
   if (chunk === null) {
     er = new TypeError('May not write null values to stream');
-  } else if (!Buffer.isBuffer(chunk) && typeof chunk !== 'string' && chunk !== undefined && !state.objectMode) {
+  } else if (typeof chunk !== 'string' && chunk !== undefined && !state.objectMode) {
     er = new TypeError('Invalid non-string/buffer chunk');
   }
   if (er) {
@@ -259,19 +269,20 @@ function validChunk(stream, state, chunk, cb) {
 Writable.prototype.write = function (chunk, encoding, cb) {
   var state = this._writableState;
   var ret = false;
+  var isBuf = Buffer.isBuffer(chunk);
 
   if (typeof encoding === 'function') {
     cb = encoding;
     encoding = null;
   }
 
-  if (Buffer.isBuffer(chunk)) encoding = 'buffer';else if (!encoding) encoding = state.defaultEncoding;
+  if (isBuf) encoding = 'buffer';else if (!encoding) encoding = state.defaultEncoding;
 
   if (typeof cb !== 'function') cb = nop;
 
-  if (state.ended) writeAfterEnd(this, cb);else if (validChunk(this, state, chunk, cb)) {
+  if (state.ended) writeAfterEnd(this, cb);else if (isBuf || validChunk(this, state, chunk, cb)) {
     state.pendingcb++;
-    ret = writeOrBuffer(this, state, chunk, encoding, cb);
+    ret = writeOrBuffer(this, state, isBuf, chunk, encoding, cb);
   }
 
   return ret;
@@ -311,10 +322,11 @@ function decodeChunk(state, chunk, encoding) {
 // if we're already writing something, then just put this
 // in the queue, and wait our turn.  Otherwise, call _write
 // If we return false, then we need a drain event, so set that flag.
-function writeOrBuffer(stream, state, chunk, encoding, cb) {
-  chunk = decodeChunk(state, chunk, encoding);
-
-  if (Buffer.isBuffer(chunk)) encoding = 'buffer';
+function writeOrBuffer(stream, state, isBuf, chunk, encoding, cb) {
+  if (!isBuf) {
+    chunk = decodeChunk(state, chunk, encoding);
+    if (Buffer.isBuffer(chunk)) encoding = 'buffer';
+  }
   var len = state.objectMode ? 1 : chunk.length;
 
   state.length += len;
@@ -383,8 +395,8 @@ function onwrite(stream, er) {
       asyncWrite(afterWrite, stream, state, finished, cb);
       /*</replacement>*/
     } else {
-        afterWrite(stream, state, finished, cb);
-      }
+      afterWrite(stream, state, finished, cb);
+    }
   }
 }
 
@@ -535,7 +547,6 @@ function CorkedRequest(state) {
 
   this.next = null;
   this.entry = null;
-
   this.finish = function (err) {
     var entry = _this.entry;
     _this.entry = null;