Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 19 additions & 3 deletions lib/internal/streams/readable.js
Original file line number Diff line number Diff line change
Expand Up @@ -428,6 +428,10 @@ Readable.prototype.push = function(chunk, encoding) {
state.length = 0;
readableAddChunkPushByteMode(this, state, first);
}
} else if (state.fastChunk != null) {
// A push() from flowSync()'s 'data' event goes behind the prefetched
// chunk, and flowSync() must see that the buffer is in use.
materializeFastChunk(state);
}

debug('push', chunk);
Expand All @@ -441,6 +445,12 @@ Readable.prototype.push = function(chunk, encoding) {
Readable.prototype.unshift = function(chunk, encoding) {
debug('unshift', chunk);
const state = this._readableState;
// An unshift() from flowSync()'s 'data' event or _read() goes in front of
// the prefetched chunk, and flowSync() must see that the buffer is in use.
if ((state[kState] & kFastPush) !== 0 || state.fastChunk != null) {
state[kState] &= ~kFastPush;
materializeFastChunk(state);
}
return (state[kState] & kObjectMode) === 0 ?
readableAddChunkUnshiftByteMode(this, state, chunk, encoding) :
readableAddChunkUnshiftObjectMode(this, state, chunk);
Expand Down Expand Up @@ -630,6 +640,8 @@ Readable.prototype.isPaused = function() {
// Backwards compatibility.
Readable.prototype.setEncoding = function(enc) {
const state = this._readableState;
// A push() that follows a setEncoding() from _read() has to be decoded.
state[kState] &= ~kFastPush;
if (state.fastChunk != null)
materializeFastChunk(state);

Expand Down Expand Up @@ -698,8 +710,8 @@ function howMuchToRead(n, state) {

// You can override either this method, or the async _read(n) below.
Readable.prototype.read = function(n) {
// A nested read() during flowSync()'s 'data' event must see the
// prefetched chunk. Null for every read that is not inside that loop.
// A nested read() during flowSync()'s 'data' event or _read() must see
// the prefetched chunk. Null for every read that is not inside that loop.
if (this._readableState.fastChunk != null)
materializeFastChunk(this._readableState);

Expand Down Expand Up @@ -1472,7 +1484,8 @@ function flowSync(stream, state) {
deliverFlowChunk(stream, state, current);
}

// Nested read() moved fastChunk into the buffer and may have refilled.
// A nested read(), unshift() or push() moved fastChunk into the buffer
// and may have added to it.
if (state.fastChunk == null && state.length !== 0)
break;
}
Expand All @@ -1485,11 +1498,14 @@ function flowSync(stream, state) {
}

// Put fastChunk at the head of the buffer. state.length already counts it.
// The buffer is in use from here on, so a later push() from the same _read()
// must add to it instead of replacing fastChunk and state.length.
function materializeFastChunk(state) {
const chunk = state.fastChunk;
if (chunk == null)
return;
state.fastChunk = null;
state[kState] &= ~kFastPush;
if (state.bufferIndex > 0) {
state.buffer[--state.bufferIndex] = chunk;
} else {
Expand Down
139 changes: 139 additions & 0 deletions test/parallel/test-stream-readable-flow-sync-buffered.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,139 @@
'use strict';

// A flowing byte-mode Readable that pushes from _read() keeps the prefetched
// chunk off the buffer. Data put on the buffer by unshift(), push() or a
// nested read() while that chunk is pending must not be lost or reordered,
// and a setEncoding() from _read() must apply to the chunks pushed after it.

const common = require('../common');
const assert = require('assert');
const { Readable } = require('stream');

function source(chunks, read) {
return new Readable({
read: read ?? function() {
this.push(chunks.shift() ?? null);
},
});
}

function collect(stream, ondata, expected) {
const received = [];
let first = true;
stream.on('data', (chunk) => {
if (first) {
first = false;
ondata(chunk, received);
} else {
received.push(chunk);
}
});
stream.on('end', common.mustCall(() => {
assert.strictEqual(Buffer.concat(received).toString(), expected);
assert.strictEqual(stream.readableLength, 0);
}));
}

// unshift() from the 'data' listener.
{
const stream = source([Buffer.from('AB'), Buffer.from('CD'), Buffer.from('EF')]);
collect(stream, common.mustCall((chunk, received) => {
received.push(chunk.subarray(0, 1));
stream.unshift(chunk.subarray(1));
assert.strictEqual(stream.readableLength, 3);
}), 'ABCDEF');
}

// push() from the 'data' listener.
{
const stream = source([Buffer.from('AB'), Buffer.from('CD'), Buffer.from('EF')]);
collect(stream, common.mustCall((chunk, received) => {
received.push(chunk);
stream.push(Buffer.from('X'));
assert.strictEqual(stream.readableLength, 3);
}), 'ABCDXEF');
}

// unshift() from _read(), before and after push(). Every chunk must be
// delivered. The order is not checked.
{
const chunks = [Buffer.from('AB'), Buffer.from('CD'), Buffer.from('EF')];
let reads = 0;
const stream = source(chunks, function() {
reads++;
if (reads === 2)
this.unshift(Buffer.from('X'));
this.push(chunks.shift() ?? null);
if (reads === 3)
this.unshift(Buffer.from('Y'));
});
const received = [];
stream.on('data', (chunk) => received.push(...chunk.toString()));
stream.on('end', common.mustCall(() => {
assert.strictEqual(received.sort().join(''), 'ABCDEFXY');
assert.strictEqual(stream.readableLength, 0);
}));
}

// read(0) from _read() after push(). The nested _read() adds to the buffer.
{
const chunks = [Buffer.from('AB'), Buffer.from('CD'), Buffer.from('EF'),
Buffer.from('GH')];
let reads = 0;
const stream = source(chunks, function() {
reads++;
this.push(chunks.shift() ?? null);
if (reads === 2)
this.read(0);
});
const received = [];
stream.on('data', (chunk) => received.push(chunk));
stream.on('end', common.mustCall(() => {
assert.strictEqual(Buffer.concat(received).toString(), 'ABCDEFGH');
assert.strictEqual(stream.readableLength, 0);
}));
}

// read(0) from _read() between two push() calls.
{
const chunks = [Buffer.from('AB'), Buffer.from('CD'), Buffer.from('EF'),
Buffer.from('GH'), Buffer.from('IJ')];
let reads = 0;
const stream = source(chunks, function() {
reads++;
this.push(chunks.shift() ?? null);
if (reads === 2) {
this.read(0);
this.push(chunks.shift() ?? null);
}
});
const received = [];
stream.on('data', (chunk) => received.push(chunk));
stream.on('end', common.mustCall(() => {
assert.strictEqual(Buffer.concat(received).toString(), 'ABCDEFGHIJ');
assert.strictEqual(stream.readableLength, 0);
}));
}

// setEncoding() from _read(), before and after push(). The chunk delivered
// by the 'data' event in progress was read before the call and stays a
// Buffer. Every later chunk is a decoded string.
for (const before of [true, false]) {
const chunks = [Buffer.from('AB'), Buffer.from('CD'), Buffer.from('EF')];
let reads = 0;
const stream = source(chunks, function() {
reads++;
if (reads === 2 && before)
this.setEncoding('hex');
this.push(chunks.shift() ?? null);
if (reads === 2 && !before)
this.setEncoding('hex');
});
const received = [];
stream.on('data', (chunk) => received.push(chunk));
stream.on('end', common.mustCall(() => {
assert.deepStrictEqual(received[0], Buffer.from('AB'));
assert.strictEqual(received.slice(1).join(''), '43444546');
assert.strictEqual(stream.readableLength, 0);
}));
}
Loading