-
-
Notifications
You must be signed in to change notification settings - Fork 36.7k
Expand file tree
/
Copy pathcompare-node-bench.js
More file actions
118 lines (108 loc) · 4.21 KB
/
Copy pathcompare-node-bench.js
File metadata and controls
118 lines (108 loc) · 4.21 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
'use strict';
const path = require('node:path');
const CLI = require('./_cli.js');
const { analyzeCompare } = require('./_node-bench-analysis.js');
const {
csvEncode,
parseInteger,
parseNumber,
runBenchmark,
} = require('./_node-bench.js');
const cli = new CLI(`usage: ./node compare-node-bench.js [options] [--] <file> ...
Run explicit node:bench files repeatedly with two Node.js binaries. Each
observation runs in a fresh process. Output is compatible with compare.R,
or --analyze can summarize it directly.
--new binary new Node.js binary (required)
--old binary old Node.js binary (required)
--runs 30 observations per binary
--warmup 0 warmup samples before each observation
--name-pattern pattern only run matching benchmarks
--node-arg argument pass an argument to both binaries (repeatable)
--analyze analyze with Welch's t-test instead of writing CSV
--scale 1000 rate multiplier used for histogram precision
--max-regression N fail if a family-wise significant regression's
95% confidence interval is entirely beyond N%
(implies --analyze)
`, { arrayArgs: ['node-arg'], boolArgs: ['analyze'] });
if (!cli.optional.new || !cli.optional.old || cli.items.length === 0) {
cli.abort(cli.usage);
}
async function main() {
const runs = parseInteger(cli.optional.runs, 30, '--runs', 1);
const warmup = parseInteger(cli.optional.warmup, 0, '--warmup', 0);
const scale = parseInteger(cli.optional.scale, 1000, '--scale', 1);
const hasMaxRegression = cli.optional['max-regression'] !== undefined;
const maxRegression = parseNumber(
cli.optional['max-regression'], 0, '--max-regression', 0);
const analyze = !!cli.optional.analyze || hasMaxRegression;
const options = {
namePattern: cli.optional['name-pattern'],
nodeArgs: cli.optional['node-arg'],
warmup,
};
const binaries = [
{ label: 'old', path: cli.optional.old },
{ label: 'new', path: cli.optional.new },
];
const rows = [];
const counts = new Map();
const csvGroups = new Map();
for (const file of cli.items) {
const resolved = path.resolve(file);
for (let run = 0; run < runs; run++) {
const order = run % 2 === 0 ? binaries : [binaries[1], binaries[0]];
for (const binary of order) {
const samples = await runBenchmark(binary.path, resolved, options);
for (const sample of samples) {
const identity = JSON.stringify([resolved, sample.identity]);
const csvGroup = JSON.stringify([
sample.name,
sample.configuration,
]);
const groupedIdentity = csvGroups.get(csvGroup);
if (groupedIdentity !== undefined && groupedIdentity !== identity) {
throw new Error(
`Distinct benchmarks would share the CSV group '${sample.name} ` +
`${sample.configuration}'`,
);
}
csvGroups.set(csvGroup, identity);
let count = counts.get(identity);
if (count === undefined) {
count = { name: sample.name, new: 0, old: 0 };
counts.set(identity, count);
}
count[binary.label]++;
rows.push({ binary: binary.label, ...sample });
}
}
}
}
if (rows.length === 0) {
throw new Error('No benchmark samples were produced');
}
for (const count of counts.values()) {
if (count.old !== runs || count.new !== runs) {
throw new Error(
`Benchmark '${count.name}' was not reported by both binaries in every run`,
);
}
}
if (analyze) {
const result = analyzeCompare(
rows, scale, hasMaxRegression ? maxRegression : undefined);
process.stdout.write(result.output);
if (result.failed) process.exitCode = 1;
return;
}
const output = ['"binary","filename","configuration","rate","time"'];
for (const row of rows) {
output.push(`${csvEncode(row.binary)},${csvEncode(row.name)},` +
`${csvEncode(row.configuration)},${row.rate},${row.duration}`);
}
process.stdout.write(`${output.join('\n')}\n`);
}
main().catch((error) => {
console.error(error.stack);
process.exitCode = 1;
});