-
-
Notifications
You must be signed in to change notification settings - Fork 36.7k
Expand file tree
/
Copy pathcompare.js
More file actions
444 lines (388 loc) · 16.5 KB
/
Copy pathcompare.js
File metadata and controls
444 lines (388 loc) · 16.5 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
'use strict';
const { spawn, fork } = require('node:child_process');
const { closeSync, openSync, writeSync } = require('node:fs');
const { inspect } = require('util');
const path = require('path');
const CLI = require('./_cli.js');
const BenchmarkProgress = require('./_benchmark_progress.js');
//
// Parse arguments
//
const cli = new CLI(`usage: ./node compare.js [options] [--] <category> ...
Run each benchmark in the <category> directory many times using two different
node versions. More than one <category> directory can be specified.
The output is formatted as csv, which can be processed using for
example 'compare.R'. Use --analyze to perform statistical analysis
directly without R.
--new ./new-node-binary new node binary (required)
--old ./old-node-binary old node binary (required)
--runs 30 number of samples
--filter pattern includes only benchmark scripts matching
<pattern> (can be repeated)
--exclude pattern excludes scripts matching <pattern> (can be
repeated)
--set variable=value set benchmark variable (can be repeated)
--no-progress don't show benchmark progress indicator
--analyze perform statistical analysis after benchmarks
complete (Welch's t-test, effect size) instead
of printing csv output to stdout
--csv filename write csv output to filename (can be combined
with --analyze). Use - to write to stdout.
--scale 1000 rate-to-integer multiplier for histogram
precision when using --analyze (default: 1000)
--max-regression N exit with code 1 if any statistically
significant regression exceeds N% (implies
--analyze)
Examples:
--set CPUSET=0 Runs benchmarks on CPU core 0.
--set CPUSET=0-2 Specifies that benchmarks should run on CPU cores 0 to 2.
Note: The CPUSET format should match the specifications of the 'taskset' command
`, { arrayArgs: ['set', 'filter', 'exclude'], boolArgs: ['no-progress', 'analyze'] });
if (!cli.optional.new || !cli.optional.old) {
cli.abort(cli.usage);
}
const binaries = ['old', 'new'];
const runs = cli.optional.runs ? parseInt(cli.optional.runs, 10) : 30;
const maxRegression = cli.optional['max-regression'] ?
parseFloat(cli.optional['max-regression']) :
0;
const analyze = !!cli.optional.analyze || maxRegression > 0;
const scale = cli.optional.scale ? parseInt(cli.optional.scale, 10) : 1000;
const benchmarks = cli.benchmarks();
if (benchmarks.length === 0) {
console.error('No benchmarks found');
process.exitCode = 1;
return;
}
const cvsToStdout = cli.optional.csv === '-';
const csvFd = cli.optional.csv === undefined || cvsToStdout ?
null :
openSync(cli.optional.csv, 'w');
const outputCsv = !analyze || csvFd !== null || cvsToStdout;
function writeCsv(line) {
writeSync(csvFd || process.stdout.fd, `${line}\n`);
}
// When --analyze is set, collect results for statistical analysis.
const results = analyze ? new Map() : null;
// Create queue from the benchmarks list such both node versions are tested
// `runs` amount of times each.
// Note: BenchmarkProgress relies on this order to estimate
// how much runs remaining for a file. All benchmarks generated from
// the same file must be run consecutively.
const queue = [];
for (const filename of benchmarks) {
for (let iter = 0; iter < runs; iter++) {
for (const binary of binaries) {
queue.push({ binary, filename, iter });
}
}
}
// queue.length = binary.length * runs * benchmarks.length
// Print csv header unless only analyzing inline.
if (outputCsv) {
writeCsv('"binary","filename","configuration","rate","time"');
}
const kStartOfQueue = 0;
const showProgress = !cli.optional['no-progress'] && !cvsToStdout;
let progress;
if (showProgress) {
progress = new BenchmarkProgress(queue, benchmarks, {
analyze: analyze || csvFd !== null,
});
progress.startQueue(kStartOfQueue);
}
(function recursive(i) {
const job = queue[i];
const resolvedPath = path.resolve(__dirname, job.filename);
const cpuCore = cli.getCpuCoreSetting();
let child;
if (cpuCore !== null) {
const spawnArgs = ['-c', cpuCore, cli.optional[job.binary], resolvedPath, ...cli.optional.set];
child = spawn('taskset', spawnArgs, {
env: process.env,
stdio: ['inherit', 'inherit', 'inherit', 'ipc'],
});
} else {
child = fork(resolvedPath, cli.optional.set, {
execPath: cli.optional[job.binary],
});
}
child.on('message', (data) => {
if (data.type === 'report') {
// Construct configuration string, " A=a, B=b, ..."
let conf = '';
for (const key of Object.keys(data.conf)) {
conf += ` ${key}=${inspect(data.conf[key])}`;
}
conf = conf.slice(1);
if (analyze) {
// Collect results for post-run analysis.
const name = `${job.filename} ${conf}`;
if (!results.has(name)) {
results.set(name, { old: [], new: [] });
}
results.get(name)[job.binary].push(data.rate);
}
if (outputCsv) {
// Escape quotes (") for correct csv formatting
const csvConf = conf.replace(/"/g, '""');
writeCsv(`"${job.binary}","${job.filename}","${csvConf}",` +
`${data.rate},${data.time}`);
}
if (showProgress) {
// One item in the subqueue has been completed.
progress.completeConfig(data);
}
} else if (showProgress && data.type === 'config') {
// The child has computed the configurations, ready to run subqueue.
progress.startSubqueue(data, i);
}
});
child.once('close', (code) => {
if (code) {
process.exit(code);
}
if (showProgress) {
progress.completeRun(job);
}
// If there are more benchmarks execute the next
if (i + 1 < queue.length) {
recursive(i + 1);
} else {
if (csvFd !== null) closeSync(csvFd);
if (analyze) printAnalysis(results, scale, maxRegression);
}
});
})(kStartOfQueue);
// Holm-Bonferroni step-down adjustment. Controls the probability of *any*
// false positive across the whole comparison set, which is what a pass/fail
// gate needs: an uncorrected suite of 169 comparisons at 5% has a 99.98%
// chance of flagging something that is not there. Uniformly more powerful
// than plain Bonferroni, and makes no assumption about independence.
function holmAdjust(pValues) {
const order = pValues
.map((p, i) => ({ p, i }))
.sort((a, b) => a.p - b.p);
const m = order.length;
const adjusted = new Array(m);
let running = 0;
for (let k = 0; k < m; k++) {
// Step down, enforcing monotonicity so an adjusted value can never be
// smaller than one belonging to a more significant raw p-value.
running = Math.max(running, Math.min(1, (m - k) * order[k].p));
adjusted[order[k].i] = running;
}
return adjusted;
}
function printAnalysis(results, scale, maxRegression) {
const { createHistogram } = require('node:perf_hooks');
// Build per-benchmark histograms and run statistical tests.
const rows = [];
let maxNameLen = 0;
let skipped = 0;
for (const [name, { old: oldRates, new: newRates }] of results) {
if (oldRates.length < 2 || newRates.length < 2) {
skipped++;
continue;
}
const hOld = createHistogram({ figures: 3 });
const hNew = createHistogram({ figures: 3 });
for (const r of oldRates) hOld.record(Math.max(1, Math.round(r * scale)));
for (const r of newRates) hNew.record(Math.max(1, Math.round(r * scale)));
const oldMean = oldRates.reduce((a, b) => a + b, 0) / oldRates.length;
const newMean = newRates.reduce((a, b) => a + b, 0) / newRates.length;
const improvement = ((newMean - oldMean) / oldMean) * 100;
// Query the three confidence levels. The p-value and t-statistic
// are the same regardless of the confidence level, so we extract
// them from the first result.
const w95 = hOld.welchTest(hNew, { confidence: 0.95 });
const w99 = hOld.welchTest(hNew, { confidence: 0.99 });
const w999 = hOld.welchTest(hNew, { confidence: 0.999 });
// Significance stars matching compare.R convention.
let stars = '';
if (w95.pValue < 0.001) stars = '***';
else if (w95.pValue < 0.01) stars = ' **';
else if (w95.pValue < 0.05) stars = ' *';
// Confidence intervals expressed as percentage of the old mean.
const ciPct = (w) => {
const half =
(w.confidenceInterval.upper - w.confidenceInterval.lower) / 2;
return (half / (oldMean * scale)) * 100;
};
rows.push({
name,
stars,
improvement,
ci95: ciPct(w95),
ci99: ciPct(w99),
ci999: ciPct(w999),
pValue: w95.pValue,
});
if (name.length > maxNameLen) maxNameLen = name.length;
}
// Adjust for the size of the comparison set. The raw p-value answers "is
// this one benchmark different", but a suite is read as a whole, so the
// relevant question is "is anything here different".
const adjusted = holmAdjust(rows.map((r) => r.pValue));
for (let i = 0; i < rows.length; i++) rows[i].pAdjusted = adjusted[i];
// A comparison can only rule out an effect it was able to resolve. Where no
// threshold has been given there is no definition of "worth detecting", so
// nothing is claimed. `maxRegression` is exactly such a declaration, so it
// is reused rather than inventing a second constant.
const resolution = maxRegression > 0 ? maxRegression : null;
let underpowered = 0;
for (const row of rows) {
row.inconclusive = resolution !== null &&
row.stars.trim() === '' &&
row.ci95 > resolution;
if (row.inconclusive) underpowered++;
}
// Print header.
const pad = (s, n) => s + ' '.repeat(Math.max(0, n - s.length));
const rpad = (s, n) => ' '.repeat(Math.max(0, n - s.length)) + s;
writeSync(process.stdout.fd, `${pad('', maxNameLen)} confidence` +
` improvement accuracy (*) (**) (***)\n`);
for (const row of rows) {
const imp = `${row.improvement >= 0 ? '+' : ''}${row.improvement.toFixed(2)} %`;
writeSync(process.stdout.fd,
`${pad(row.name, maxNameLen)} ${pad(row.stars, 10)}` +
` ${rpad(imp, 11)}` +
` ±${row.ci95.toFixed(2)}%` +
` ±${row.ci99.toFixed(2)}%` +
` ±${row.ci999.toFixed(2)}%` +
`${row.inconclusive ? ' (inconclusive)' : ''}\n`,
);
}
if (skipped > 0) {
writeSync(process.stdout.fd, '\n');
writeSync(process.stdout.fd,
`Note: ${skipped} configuration${skipped === 1 ? ' was' : 's were'}` +
` skipped because Welch's t-test requires at least 2 samples per` +
` binary. Use --runs 2 or higher.\n`,
);
}
// --- Bar chart visualization ---
printChart(rows, maxNameLen);
writeSync(process.stdout.fd, '\n');
writeSync(process.stdout.fd,
`Rates were scaled by ${scale}x into HdrHistogram (3 significant figures).\n` +
`Use --scale to adjust precision if needed.\n\n`,
);
const anyFamilyWise = rows.filter((r) => r.pAdjusted < 0.05).length;
writeSync(process.stdout.fd,
`Be aware that when doing many comparisons the risk of a false-positive\n` +
`result increases. In this case, there are ${rows.length} comparisons, ` +
`you can thus\nexpect the following amount of false-positive results:\n` +
` ${(rows.length * 0.05).toFixed(2)} false positives, when considering ` +
`a 5% risk acceptance (*, **, ***),\n` +
` ${(rows.length * 0.01).toFixed(2)} false positives, when considering ` +
`a 1% risk acceptance (**, ***),\n` +
` ${(rows.length * 0.001).toFixed(2)} false positives, when considering ` +
`a 0.1% risk acceptance (***)\n` +
`\nThe stars above are per-benchmark and uncorrected. Adjusting for the ` +
`size of\nthis comparison set (Holm-Bonferroni), ${anyFamilyWise} ` +
`comparison${anyFamilyWise === 1 ? '' : 's'} remain${anyFamilyWise === 1 ? 's' : ''} ` +
`significant at 5%.\n--max-regression uses the corrected values.\n`,
);
// Gate: exit with error if any regression is shown to exceed the limit.
if (maxRegression > 0) {
if (underpowered > 0) {
writeSync(process.stdout.fd, '\n');
writeSync(process.stdout.fd,
`Note: ${underpowered} of ${rows.length} comparison` +
`${rows.length === 1 ? '' : 's'} could not resolve an effect as ` +
`small as ${maxRegression}%, and are marked (inconclusive). They are ` +
`not\nevidence of no regression -- the samples are too noisy to tell. ` +
`Raise --runs,\nor pin cores with --set CPUSET, to narrow them.\n`,
);
}
// Two conditions, both required.
//
// The confidence interval must lie entirely beyond the threshold. A small
// p-value only says the effect is not exactly zero; claiming it exceeds
// `maxRegression` is a statement about magnitude, so the interval has to
// exclude that magnitude. Testing the point estimate instead systematically
// fires on the noisiest benchmarks, because a large point estimate is
// easiest to obtain when the interval is wide.
//
// The p-value must also survive adjustment for the size of the comparison
// set, so that a suite of hundreds of benchmarks does not fail purely
// because one of them drifted.
const failures = rows.filter(
(r) => r.pAdjusted < 0.05 && r.improvement + r.ci95 < -maxRegression,
);
if (failures.length > 0) {
writeSync(process.stdout.fd, '\n');
writeSync(process.stdout.fd,
`FAIL: ${failures.length} benchmark${failures.length === 1 ? '' : 's'}` +
` regressed by more than ${maxRegression}%` +
` (interval excludes the threshold,\n` +
`family-wise corrected across ${rows.length} comparisons):\n`,
);
for (const f of failures) {
writeSync(process.stdout.fd,
` ${f.name} ${f.improvement.toFixed(2)}% ` +
`(95% CI up to ${(f.improvement + f.ci95).toFixed(2)}%, ` +
`adjusted p=${f.pAdjusted.toExponential(2)})\n`,
);
}
process.exitCode = 1;
}
}
}
function printChart(rows, maxNameLen) {
if (rows.length === 0) return;
// Determine the chart scale from the data. The bar region covers
// the range [-maxAbs, +maxAbs] so the zero line sits in the center.
const barWidth = 40;
const halfWidth = barWidth / 2;
let maxAbs = 0;
for (const row of rows) {
const extent = Math.abs(row.improvement) + row.ci95;
if (extent > maxAbs) maxAbs = extent;
}
if (maxAbs === 0) maxAbs = 1;
const pad = (s, n) => s + ' '.repeat(Math.max(0, n - s.length));
// Scale axis labels.
const axisLeft = `-${maxAbs.toFixed(1)}%`;
const axisRight = `+${maxAbs.toFixed(1)}%`;
const axisCenter = '0%';
// Print axis header.
const labelPad = maxNameLen + 5;
const leftLabel = ' '.repeat(labelPad) +
axisLeft +
' '.repeat(Math.max(0, halfWidth - axisLeft.length - Math.floor(axisCenter.length / 2))) +
axisCenter +
' '.repeat(Math.max(0, halfWidth - Math.ceil(axisCenter.length / 2) - axisRight.length)) +
axisRight;
writeSync(process.stdout.fd, '\n');
writeSync(process.stdout.fd, `${leftLabel}\n`);
for (const row of rows) {
const imp = row.improvement;
const ci = row.ci95;
// Position of the improvement value in the bar region [0, barWidth].
const center = halfWidth;
const impPos = center + (imp / maxAbs) * halfWidth;
// CI extent in bar positions.
const ciLeft = center + ((imp - ci) / maxAbs) * halfWidth;
const ciRight = center + ((imp + ci) / maxAbs) * halfWidth;
// Build the bar character by character.
const chars = [];
for (let x = 0; x < barWidth; x++) {
const pos = x + 0.5; // Center of this character cell.
if (x === Math.floor(center)) {
chars.push('|');
} else if ((imp >= 0 && pos > center && pos <= impPos) ||
(imp < 0 && pos < center && pos >= impPos)) {
chars.push(row.stars ? '\u2588' : '\u2593'); // solid or dark shade
} else if (pos >= ciLeft && pos <= ciRight) {
chars.push('\u2591'); // Light shade for CI region
} else {
chars.push(' ');
}
}
const label = `${row.improvement >= 0 ? '+' : ''}${row.improvement.toFixed(2)}%`;
const sig = row.stars.trim();
writeSync(process.stdout.fd, `${pad(row.name, maxNameLen)} ${chars.join('')} ${label} ${sig}\n`);
}
}