Skip to content

Commit ecde823

Browse files
committed
Update benchmark to mitata
1 parent 23c62cd commit ecde823

2 files changed

Lines changed: 73 additions & 35 deletions

File tree

‎benchmark/benchmark.mjs‎

Lines changed: 72 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -1,53 +1,74 @@
11
#!/usr/bin/env node
2-
import { readFileSync, readdirSync } from 'fs'
2+
/* eslint no-console:0 */
3+
4+
import fs from 'node:fs'
35
import util from 'node:util'
4-
import Benchmark from 'benchmark'
5-
import ansi from 'ansi'
6-
const cursor = ansi(process.stdout)
6+
import { do_not_optimize as doNotOptimize, measure } from 'mitata'
7+
8+
const WARMUP_MSEC = 250
9+
const BENCHMARK_OPTIONS = {
10+
/* min_cpu_time: 1000 * 1e6,
11+
min_samples: 24 */
12+
}
713

814
const IMPLS = []
915

10-
for (const name of readdirSync(new URL('./implementations', import.meta.url)).sort()) {
16+
for (const name of fs.readdirSync(new URL('./implementations', import.meta.url)).sort()) {
1117
const filepath = new URL(`./implementations/${name}/index.mjs`, import.meta.url)
12-
const code = (await import(filepath))
18+
const code = await import(filepath)
1319

1420
IMPLS.push({ name, code })
1521
}
1622

1723
const SAMPLES = []
1824

19-
readdirSync(new URL('./samples', import.meta.url)).sort().forEach(sample => {
25+
fs.readdirSync(new URL('./samples', import.meta.url)).sort().forEach((sample) => {
2026
const filepath = new URL(`./samples/${sample}`, import.meta.url)
2127

2228
const content = {}
2329

24-
content.string = readFileSync(filepath, 'utf8')
30+
content.string = fs.readFileSync(filepath, 'utf8')
2531

2632
const title = `(${content.string.length} bytes)`
2733

28-
function onComplete () {
29-
cursor.write('\n')
30-
}
34+
SAMPLES.push({ name: sample.split('.')[0], filename: sample, title, content })
35+
})
3136

32-
const suite = new Benchmark.Suite(title, {
33-
onStart: () => { console.log('\nSample: %s %s', sample, title) },
34-
onComplete
35-
})
37+
function formatNumber (num) {
38+
return num.toLocaleString('en-US', { maximumFractionDigits: 0 })
39+
}
3640

37-
IMPLS.forEach(function (impl) {
38-
suite.add(impl.name, {
39-
onCycle: event => {
40-
cursor.horizontalAbsolute()
41-
cursor.eraseLine()
42-
cursor.write(` > ${event.target}`)
43-
},
44-
onComplete,
45-
fn: () => { impl.code.run(content.string) }
46-
})
47-
})
41+
function relativeStandardError (samples, mean) {
42+
if (samples.length < 2 || mean === 0) return 0
4843

49-
SAMPLES.push({ name: sample.split('.')[0], title, content, suite })
50-
})
44+
const variance = samples.reduce((acc, sample) => {
45+
return acc + (sample - mean) ** 2
46+
}, 0) / (samples.length - 1)
47+
48+
return (Math.sqrt(variance) / Math.sqrt(samples.length) / mean) * 100
49+
}
50+
51+
function formatRun (run) {
52+
if (run.error) return `${run.name}: error`
53+
54+
const stats = run.stats
55+
const throughput = 1e9 / stats.avg
56+
57+
return [
58+
run.name,
59+
`${formatNumber(throughput)} ops/sec`,
60+
`+/-${relativeStandardError(stats.samples, stats.avg).toFixed(2)}%`,
61+
`${formatNumber(stats.ticks)} samples`
62+
].join(' ')
63+
}
64+
65+
function warmup (impl, data) {
66+
const started = performance.now()
67+
68+
do {
69+
doNotOptimize(impl.code.run(data))
70+
} while (performance.now() - started < WARMUP_MSEC)
71+
}
5172

5273
function select (patterns) {
5374
const result = []
@@ -71,7 +92,7 @@ function select (patterns) {
7192
return result
7293
}
7394

74-
function run (files) {
95+
async function run (files) {
7596
const selected = select(files)
7697

7798
if (selected.length > 0) {
@@ -83,11 +104,29 @@ function run (files) {
83104
console.log('There isn\'t any sample matches any of these patterns: %s', util.inspect(files))
84105
}
85106

86-
selected.forEach(function (sample) {
87-
sample.suite.run()
88-
})
107+
for (const sample of selected) {
108+
console.log('\n\nSample: %s %s', sample.filename, sample.title)
109+
110+
IMPLS.forEach((impl) => {
111+
warmup(impl, sample.content.string)
112+
})
113+
114+
for (const impl of IMPLS) {
115+
let stats, error
116+
117+
try {
118+
stats = await measure(() => {
119+
doNotOptimize(impl.code.run(sample.content.string))
120+
}, BENCHMARK_OPTIONS)
121+
} catch (err) {
122+
error = err
123+
}
124+
125+
console.log(' > %s', formatRun({ name: impl.name, stats, error }))
126+
}
127+
}
89128
}
90129

91-
run(process.argv.slice(2).map(function (source) {
130+
await run(process.argv.slice(2).map(function (source) {
92131
return new RegExp(source, 'i')
93132
}))

‎package.json‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -51,11 +51,10 @@
5151
},
5252
"devDependencies": {
5353
"@rollup/plugin-node-resolve": "^15.2.3",
54-
"ansi": "^0.3.0",
55-
"benchmark": "^2.1.0",
5654
"c8": "^8.0.1",
5755
"eslint": "^9.39.3",
5856
"gh-pages": "^6.1.0",
57+
"mitata": "^1.0.34",
5958
"neostandard": "^0.13.0",
6059
"ndoc": "^6.0.0",
6160
"rollup": "^4.6.1",

0 commit comments

Comments
 (0)