Skip to content

Commit 209fb77

Browse files
anonrigaduh95
authored andcommitted
zlib: grow the output buffer of large async one-shot calls
The asynchronous convenience methods hand the whole input to zlib in one write, but still take one threadpool round trip per `chunkSize` (16 KiB by default) of output. Decompressing 4 MiB took 257 round trips. Only the concatenated result of these calls is observable, so double the output buffer whenever it fills up, up to 1 MiB. 4 MiB of output now takes 10 round trips. The buffer never grows past what is needed to tell that `maxOutputLength` was exceeded, and results that fit in the first buffer are unchanged. Signed-off-by: Yagiz Nizipli <yagiz@nizipli.com> Assisted-by: Grok PR-URL: #66863 Reviewed-By: Robert Nagy <ronagy@icloud.com> Reviewed-By: Matteo Collina <matteo.collina@gmail.com>
1 parent 86f2e27 commit 209fb77

3 files changed

Lines changed: 112 additions & 1 deletion

File tree

‎lib/zlib.js‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
const {
2525
ArrayBuffer,
2626
MathMax,
27+
MathMin,
2728
NumberIsNaN,
2829
ObjectDefineProperties,
2930
ObjectDefineProperty,
@@ -182,6 +183,18 @@ function zlibBufferOnData(chunk) {
182183
zlibBufferAddChunk(this, chunk);
183184
}
184185

186+
// Only the concatenated output of a one-shot call is observable, so its output
187+
// buffer can double in size whenever it fills up, and large results take far
188+
// fewer threadpool round trips. It never grows past what is needed to detect
189+
// that maxOutputLength was exceeded.
190+
const kMaxBufferResultChunkSize = 1024 * 1024;
191+
function zlibBufferGrowChunkSize(engine) {
192+
const size = MathMin(engine._chunkSize * 2, kMaxBufferResultChunkSize,
193+
engine._maxOutputLength - engine.nread + 1);
194+
if (size > engine._chunkSize)
195+
engine._chunkSize = size;
196+
}
197+
185198
function zlibBufferOnError(err) {
186199
this.removeAllListeners('end');
187200
this.cb(err);
@@ -691,6 +704,8 @@ function processCallback() {
691704

692705
// Exhausted the output buffer, or used all the input create a new one.
693706
if (availOutAfter === 0 || self._outOffset >= self._chunkSize) {
707+
if (availOutAfter === 0 && self[kBufferResult] === true)
708+
zlibBufferGrowChunkSize(self);
694709
handle.availOutBefore = self._chunkSize;
695710
self._outOffset = 0;
696711
self._outBuffer = Buffer.allocUnsafe(self._chunkSize);
Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
'use strict';
2+
// The asynchronous convenience methods grow their output buffer while
3+
// processing large inputs. Check that large results, custom chunkSize values
4+
// and maxOutputLength behave exactly like the synchronous methods.
5+
6+
const common = require('../common');
7+
const assert = require('assert');
8+
const fs = require('fs');
9+
const zlib = require('zlib');
10+
11+
const { Z_MIN_CHUNK, BROTLI_PARAM_QUALITY } = zlib.constants;
12+
13+
// Text plus pseudo-random bytes, so that both the compressed and the
14+
// decompressed output span many output buffers.
15+
const random = Buffer.alloc(256 * 1024);
16+
{
17+
let seed = 0x2545f491;
18+
for (let i = 0; i < random.length; i++) {
19+
seed ^= seed << 13;
20+
seed ^= seed >>> 17;
21+
seed ^= seed << 5;
22+
random[i] = seed & 0xff;
23+
}
24+
}
25+
const data = Buffer.concat([
26+
Buffer.alloc(2 * 1024 * 1024, fs.readFileSync(__filename)),
27+
random,
28+
]);
29+
30+
// Keep brotli fast enough for a test.
31+
const brotliOptions = { params: { [BROTLI_PARAM_QUALITY]: 4 } };
32+
const pairs = [
33+
['deflate', 'inflate'],
34+
['gzip', 'gunzip'],
35+
['gzip', 'unzip'],
36+
['deflateRaw', 'inflateRaw'],
37+
['brotliCompress', 'brotliDecompress', brotliOptions],
38+
['zstdCompress', 'zstdDecompress'],
39+
];
40+
41+
function callAsync(method, input, options) {
42+
return new Promise((resolve, reject) => {
43+
zlib[method](input, options, common.mustCall((err, result) => {
44+
if (err) reject(err);
45+
else resolve(result);
46+
}));
47+
});
48+
}
49+
50+
(async () => {
51+
for (const [compress, decompress, options = {}] of pairs) {
52+
const label = `${compress}/${decompress}`;
53+
const compressed = await callAsync(compress, data, options);
54+
const expected = zlib[`${compress}Sync`](data, options);
55+
// zstdCompress() pledges the input size and ends the frame separately,
56+
// which yields different (not larger) frames than zstdCompressSync().
57+
if (compress !== 'zstdCompress') {
58+
assert.deepStrictEqual(compressed, expected, label);
59+
}
60+
assert.deepStrictEqual(await callAsync(decompress, compressed, {}), data,
61+
label);
62+
63+
for (const chunkSize of [Z_MIN_CHUNK, data.length + 1]) {
64+
assert.deepStrictEqual(
65+
await callAsync(decompress, compressed, { chunkSize }), data,
66+
`${label} chunkSize=${chunkSize}`);
67+
}
68+
69+
for (const maxOutputLength of [data.length - 1, 100000, 1]) {
70+
await assert.rejects(
71+
callAsync(decompress, compressed, { maxOutputLength }), {
72+
name: 'RangeError',
73+
code: 'ERR_BUFFER_TOO_LARGE',
74+
message: `Cannot create a Buffer larger than ${maxOutputLength} bytes`,
75+
}, `${label} maxOutputLength=${maxOutputLength}`);
76+
}
77+
assert.deepStrictEqual(
78+
await callAsync(decompress, compressed,
79+
{ maxOutputLength: data.length }),
80+
data, label);
81+
}
82+
})().then(common.mustCall());

‎test/parallel/test-zlib-convenience-async-round-trip.js‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
'use strict';
22
// The asynchronous convenience methods know the whole input up front, so they
33
// process it with the finish flush in a single write. Check that a call whose
4-
// output fits in one output buffer takes exactly one threadpool round trip.
4+
// output fits in one output buffer takes exactly one threadpool round trip,
5+
// and that larger results need few round trips.
56

67
const common = require('../common');
78
const assert = require('assert');
@@ -71,4 +72,17 @@ function callAsync(method, buffer, options = {}) {
7172
});
7273
assert.deepStrictEqual(roundTrips, [1]);
7374
assert.strictEqual(error.code, 'Z_DATA_ERROR');
75+
76+
// The output buffer doubles from the 16 KiB default up to 1 MiB, so 4 MiB of
77+
// output takes 6 + 4 round trips instead of 256.
78+
{
79+
const large = Buffer.alloc(4 * 1024 * 1024, input);
80+
const compressed = zlib.deflateSync(large);
81+
let result;
82+
const roundTrips = await countRoundTrips(async () => {
83+
result = await callAsync('inflate', compressed);
84+
});
85+
assert.deepStrictEqual(roundTrips, [10]);
86+
assert.deepStrictEqual(result, large);
87+
}
7488
})().then(common.mustCall());

0 commit comments

Comments
 (0)