Skip to content

Commit 3a3465b

Browse files
GlassBricksGlassBricks
authored
Runtime benchmarks (#1126)
* Basic runtime benchmarks * Add basic doc for runtime benchmarks * Fix runtime benchmark text formatting * Add /benchmark/dist to .prettierignore * Sort benchmarks by name * Remove benchmark/dist/json.lua from gitignore there's already benchmark/src/json.lua * Enable benchmark in ci * Refactor comparison Markdown formatting
1 parent 09c8f0f commit 3a3465b

File tree

11 files changed

+267
-143
lines changed

11 files changed

+267
-143
lines changed

‎.eslintrc.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -218,7 +218,7 @@ module.exports = {
218218
},
219219
},
220220
{
221-
files: "benchmark/src/memory_benchmarks/**/*.ts",
221+
files: "benchmark/src/*_benchmarks/**/*.ts",
222222
rules: {
223223
"import/no-default-export": "off",
224224
},

‎.github/workflows/ci.yml‎

Lines changed: 77 additions & 77 deletions
Original file line numberDiff line numberDiff line change
@@ -39,80 +39,80 @@ jobs:
3939
- if: matrix.os == 'ubuntu-latest'
4040
uses: codecov/codecov-action@v1
4141

42-
# benchmark:
43-
# name: Benchmark
44-
# runs-on: ubuntu-latest
45-
# steps:
46-
# - name: Lua Install
47-
# run: sudo apt-get install lua5.3 luajit
48-
# - name: Glow Install
49-
# run: brew install glow
50-
# # Checkout master & commit
51-
# - name: Checkout master
52-
# uses: actions/checkout@v2
53-
# with:
54-
# ref: master
55-
# path: master
56-
# - name: Checkout commit
57-
# uses: actions/checkout@v2
58-
# with:
59-
# path: commit
60-
# - name: Use Node.js 12.13.1
61-
# uses: actions/setup-node@v1
62-
# with:
63-
# node-version: 12.13.1
64-
# # NPM
65-
# - name: NPM master
66-
# run: npm ci && npm run build
67-
# working-directory: master
68-
# - name: NPM commit
69-
# run: npm ci && npm run build
70-
# working-directory: commit
71-
# # Benchmark directory setup
72-
# - name: Ensure benchmark data dir exists
73-
# run: mkdir -p ./benchmark/data
74-
# working-directory: commit
75-
# - name: Copy commit benchmark to master
76-
# run: rm -rf ./master/benchmark && cp -rf ./commit/benchmark ./master/benchmark
77-
# # Run master benchmark first and output to commit benchmark data
78-
# - name: Build benchmark Lua 5.3 master
79-
# run: node ../../commit/dist/tstl.js -p tsconfig.53.json
80-
# working-directory: master/benchmark
81-
# - name: Run benchmark Lua 5.3 master
82-
# id: benchmark-lua-master
83-
# run: lua5.3 -- run.lua ../../../commit/benchmark/data/benchmark_master_53.json
84-
# working-directory: master/benchmark/dist
85-
# - name: Build benchmark LuaJIT master
86-
# run: node ../../commit/dist/tstl.js -p tsconfig.jit.json
87-
# working-directory: master/benchmark
88-
# - name: Run benchmark LuaJIT master
89-
# id: benchmark-jit-master
90-
# run: luajit -- run.lua ../../../commit/benchmark/data/benchmark_master_jit.json
91-
# working-directory: master/benchmark/dist
92-
# # Run commit benchmark and compare with master
93-
# - name: Build benchmark Lua 5.3 commit
94-
# run: node ../../commit/dist/tstl.js -p tsconfig.53.json
95-
# working-directory: commit/benchmark
96-
# - name: Run benchmark Lua 5.3 commit
97-
# id: benchmark-lua-commit
98-
# run: lua5.3 -- run.lua ../data/benchmark_master_vs_commit_53.json ../data/benchmark_master_53.json
99-
# working-directory: commit/benchmark/dist
100-
# - name: Build benchmark LuaJIT commit
101-
# run: node ../../commit/dist/tstl.js -p tsconfig.jit.json
102-
# working-directory: commit/benchmark
103-
# - name: Run benchmark LuaJIT commit
104-
# id: benchmark-jit-commit
105-
# run: luajit -- run.lua ../data/benchmark_master_vs_commit_jit.json ../data/benchmark_master_jit.json
106-
# working-directory: commit/benchmark/dist
107-
# - name: Combine benchmark results
108-
# id: script-combine-results
109-
# uses: actions/github-script@v3
110-
# with:
111-
# benchmark-result-path-lua: commit/benchmark/data/benchmark_master_vs_commit_53.json
112-
# benchmark-result-path-jit: commit/benchmark/data/benchmark_master_vs_commit_jit.json
113-
# result-encoding: string
114-
# script: |
115-
# const createBenchmarkCheck = require(`${process.env.GITHUB_WORKSPACE}/commit/.github/scripts/create_benchmark_check.js`);
116-
# return createBenchmarkCheck({ github, context, core });
117-
# - name: Benchmark results
118-
# run: echo "${{steps.script-combine-results.outputs.result}}" | glow -s dark -w 120 -
42+
benchmark:
43+
name: Benchmark
44+
runs-on: ubuntu-latest
45+
steps:
46+
- name: Lua Install
47+
run: sudo apt-get install lua5.3 luajit
48+
- name: Glow Install
49+
run: brew install glow
50+
# Checkout master & commit
51+
- name: Checkout master
52+
uses: actions/checkout@v2
53+
with:
54+
ref: master
55+
path: master
56+
- name: Checkout commit
57+
uses: actions/checkout@v2
58+
with:
59+
path: commit
60+
- name: Use Node.js 12.13.1
61+
uses: actions/setup-node@v1
62+
with:
63+
node-version: 12.13.1
64+
# NPM
65+
- name: NPM master
66+
run: npm ci && npm run build
67+
working-directory: master
68+
- name: NPM commit
69+
run: npm ci && npm run build
70+
working-directory: commit
71+
# Benchmark directory setup
72+
- name: Ensure benchmark data dir exists
73+
run: mkdir -p ./benchmark/data
74+
working-directory: commit
75+
- name: Copy commit benchmark to master
76+
run: rm -rf ./master/benchmark && cp -rf ./commit/benchmark ./master/benchmark
77+
# Run master benchmark first and output to commit benchmark data
78+
- name: Build benchmark Lua 5.3 master
79+
run: node ../../commit/dist/tstl.js -p tsconfig.53.json
80+
working-directory: master/benchmark
81+
- name: Run benchmark Lua 5.3 master
82+
id: benchmark-lua-master
83+
run: lua5.3 -- run.lua ../../../commit/benchmark/data/benchmark_master_53.json
84+
working-directory: master/benchmark/dist
85+
- name: Build benchmark LuaJIT master
86+
run: node ../../commit/dist/tstl.js -p tsconfig.jit.json
87+
working-directory: master/benchmark
88+
- name: Run benchmark LuaJIT master
89+
id: benchmark-jit-master
90+
run: luajit -- run.lua ../../../commit/benchmark/data/benchmark_master_jit.json
91+
working-directory: master/benchmark/dist
92+
# Run commit benchmark and compare with master
93+
- name: Build benchmark Lua 5.3 commit
94+
run: node ../../commit/dist/tstl.js -p tsconfig.53.json
95+
working-directory: commit/benchmark
96+
- name: Run benchmark Lua 5.3 commit
97+
id: benchmark-lua-commit
98+
run: lua5.3 -- run.lua ../data/benchmark_master_vs_commit_53.json ../data/benchmark_master_53.json
99+
working-directory: commit/benchmark/dist
100+
- name: Build benchmark LuaJIT commit
101+
run: node ../../commit/dist/tstl.js -p tsconfig.jit.json
102+
working-directory: commit/benchmark
103+
- name: Run benchmark LuaJIT commit
104+
id: benchmark-jit-commit
105+
run: luajit -- run.lua ../data/benchmark_master_vs_commit_jit.json ../data/benchmark_master_jit.json
106+
working-directory: commit/benchmark/dist
107+
- name: Combine benchmark results
108+
id: script-combine-results
109+
uses: actions/github-script@v3
110+
with:
111+
benchmark-result-path-lua: commit/benchmark/data/benchmark_master_vs_commit_53.json
112+
benchmark-result-path-jit: commit/benchmark/data/benchmark_master_vs_commit_jit.json
113+
result-encoding: string
114+
script: |
115+
const createBenchmarkCheck = require(`${process.env.GITHUB_WORKSPACE}/commit/.github/scripts/create_benchmark_check.js`);
116+
return createBenchmarkCheck({ github, context, core });
117+
- name: Benchmark results
118+
run: echo "${{steps.script-combine-results.outputs.result}}" | glow -s dark -w 120 -

‎.gitignore‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,4 +9,3 @@ yarn.lock
99

1010
benchmark/data/*
1111
benchmark/dist/*
12-
!benchmark/dist/json.lua

‎.prettierignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
11
/dist
22
/coverage
33
/test/translation/transformation/characterEscapeSequence.ts
4+
/benchmark/dist

‎benchmark/README.md‎

Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,12 @@
22

33
These benchmarks are written in typescript and transpiled to lua by using tstl.
44

5-
### Currently only memory benchmarks are supported
5+
### Memory benchmarks
66

77
To add a new benchmark add a new file to `memory_benchmarks`
88
and **default** export a function with the following type: `() => void`.
99
To prevent the benchmark from reporting "useful" results of your benchmark function as garbage, simply return the result.
10-
The memory used by the returned result wont count towards the total garbage amount.
10+
The memory used by the returned result won't count towards the total garbage amount.
1111

1212
For example (memory_benchmarks/my_benchmark.ts):
1313

@@ -25,19 +25,23 @@ export default function myBenchmark() {
2525
**Goal**
2626

2727
The goal of memory benchmarks is to track how much (memory) `"garbage"` is created by tstl.
28-
For that reason garabage collection is disabled in the benchmarks.
28+
For that reason garbage collection is disabled in the benchmarks.
2929

3030
You can force the creation of `"garbage"` by creating a lot of anonymous functions or temporary tables (see [lua-users.org](http://lua-users.org/wiki/OptimisingGarbageCollection) for more information).
3131

3232
To avoid crashes in the CI your benchmark should not use more than 500MB of memory.
3333

34-
**Running locally**
34+
### Runtime benchmarks
3535

36-
1. Create a benchmark baseline called "benchmark_baseline.json":
36+
To add a new runtime benchmark (execution time), add a new file to `runtime_benchmarks` and **default** export a function with the following type: `() => void`.
37+
38+
### Running locally
39+
40+
1. Create a benchmark baseline called "benchmark_baseline.json":
3741
`tstl -p tsconfig.53.json && cd dist && lua -- run.lua benchmark_baseline.json`
3842
2. Make some changes to tstl.
39-
3. Create an updated benchmark and compare with the baseline:
43+
3. Create an updated benchmark and compare with the baseline:
4044
`tstl -p tsconfig.53.json && cd dist && lua -- run.lua benchmark_updated.json benchmark_baseline.json`
4145
4. The above command will output comparison data as json to stdout.
42-
If you provide a path as third argument the comparison data will be written to that path instead.
46+
If you provide a path as third argument the comparison data will be written to that path instead.
4347
`tstl -p tsconfig.53.json && cd dist && lua -- run.lua benchmark_updated.json benchmark_baseline.json result.md`

‎benchmark/src/benchmark_types.ts‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,11 @@
11
export enum BenchmarkKind {
22
Memory = "memory",
3+
Runtime = "runtime",
34
}
45

56
export type BenchmarkFunction = () => void;
67

7-
export type BenchmarkResult = MemoryBenchmarkResult;
8+
export type BenchmarkResult = MemoryBenchmarkResult | RuntimeBenchmarkResult;
89

910
export enum MemoryBenchmarkCategory {
1011
TotalMemory = "totalMemory",
@@ -21,6 +22,16 @@ export function isMemoryBenchmarkResult(result: BenchmarkResult): result is Memo
2122
return result.kind === BenchmarkKind.Memory;
2223
}
2324

25+
export interface RuntimeBenchmarkResult {
26+
kind: BenchmarkKind.Runtime;
27+
time: number;
28+
benchmarkName: string;
29+
}
30+
31+
export function isRuntimeBenchmarkResult(result: BenchmarkResult): result is RuntimeBenchmarkResult {
32+
return result.kind === BenchmarkKind.Runtime;
33+
}
34+
2435
export interface ComparisonInfo {
2536
summary: string;
2637
text: string;

‎benchmark/src/benchmark_util.ts‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
import { BenchmarkResult } from "./benchmark_types";
2+
import { calculatePercentageChange, toFixed } from "./util";
3+
4+
export const makeMarkdownTableRow = (cells: string[]) => `| ${cells.join(" | ")} |\n`;
5+
export const makeBold = (input: string) => `**${input}**`;
6+
7+
export function compareNumericBenchmarks<T extends BenchmarkResult>(
8+
newResults: T[],
9+
oldResults: T[],
10+
unit: string,
11+
extractValue: (result: T) => number,
12+
formatValue: (value: number) => string
13+
): string {
14+
let comparisonTable = makeMarkdownTableRow([
15+
"name",
16+
`master (${unit})`,
17+
`commit (${unit})`,
18+
`change (${unit})`,
19+
"change (%)",
20+
]);
21+
comparisonTable += makeMarkdownTableRow(["---", "---", "---", "---", "---"]);
22+
23+
let oldValueSum = 0;
24+
let newValueSum = 0;
25+
26+
newResults.forEach(newResult => {
27+
const oldResult = oldResults.find(r => r.benchmarkName === newResult.benchmarkName);
28+
const newValue = extractValue(newResult);
29+
if (oldResult) {
30+
const oldValue = extractValue(oldResult);
31+
const percentageChange = calculatePercentageChange(oldValue, newValue);
32+
const change = newValue - oldValue;
33+
const row = [
34+
newResult.benchmarkName,
35+
formatValue(oldValue),
36+
formatValue(newValue),
37+
formatValue(change),
38+
toFixed(percentageChange, 2),
39+
];
40+
comparisonTable += makeMarkdownTableRow(row);
41+
oldValueSum += oldValue;
42+
newValueSum += newValue;
43+
} else {
44+
// No master found => new benchmark
45+
const row = [newResult.benchmarkName, formatValue(newValue), "/", "/", "/"];
46+
comparisonTable += makeMarkdownTableRow(row);
47+
}
48+
});
49+
50+
const sumPercentageChange = calculatePercentageChange(oldValueSum, newValueSum);
51+
comparisonTable += makeMarkdownTableRow([
52+
makeBold("sum"),
53+
makeBold(formatValue(oldValueSum)),
54+
makeBold(formatValue(newValueSum)),
55+
makeBold(formatValue(newValueSum - oldValueSum)),
56+
makeBold(toFixed(sumPercentageChange, 2)),
57+
]);
58+
59+
return comparisonTable;
60+
}

‎benchmark/src/memory_benchmark.ts‎

Lines changed: 5 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import { BenchmarkKind, MemoryBenchmarkResult, ComparisonInfo, MemoryBenchmarkCategory } from "./benchmark_types";
2-
import { toFixed, json, calculatePercentageChange } from "./util";
2+
import { compareNumericBenchmarks } from "./benchmark_util";
3+
import { toFixed, json } from "./util";
34

45
export function runMemoryBenchmark(benchmarkFunction: () => void): MemoryBenchmarkResult {
56
const result: MemoryBenchmarkResult = {
@@ -42,8 +43,6 @@ export function runMemoryBenchmark(benchmarkFunction: () => void): MemoryBenchma
4243
}
4344

4445
const formatMemory = (memInKB: number) => toFixed(memInKB / 1024, 3);
45-
const makeMarkdownTableRow = (cells: string[]) => `| ${cells.join(" | ")} |\n`;
46-
const makeBold = (input: string) => `**${input}**`;
4746

4847
export function compareMemoryBenchmarks(
4948
oldResults: MemoryBenchmarkResult[],
@@ -53,10 +52,10 @@ export function compareMemoryBenchmarks(
5352
const categories = [MemoryBenchmarkCategory.TotalMemory, MemoryBenchmarkCategory.Garbage];
5453

5554
const summary = categories
56-
.map(category => `${makeBold(category)}\n${compareCategory(newResults, oldResults, category)}`)
55+
.map(category => `### ${category}\n\n${compareCategory(newResults, oldResults, category)}`)
5756
.join("\n");
5857

59-
const text = `**master:**\n\`\`\`json\n${json.encode(oldResults)}\n\`\`\`\n**commit:**\n\`\`\`json\n${json.encode(
58+
const text = `### master:\n\`\`\`json\n${json.encode(oldResults)}\n\`\`\`\n### commit:\n\`\`\`json\n${json.encode(
6059
newResults
6160
)}\n\`\`\``;
6261

@@ -68,47 +67,5 @@ function compareCategory(
6867
oldResults: MemoryBenchmarkResult[],
6968
category: MemoryBenchmarkCategory
7069
): string {
71-
let comparisonTable = makeMarkdownTableRow(["name", "master (mb)", "commit (mb)", "change (mb)", "change (%)"]);
72-
comparisonTable += makeMarkdownTableRow(["-", "-", "-", "-", "-"]);
73-
74-
let oldValueSum = 0;
75-
let newValueSum = 0;
76-
77-
newResults.forEach(newResult => {
78-
const oldResult = oldResults.find(r => r.benchmarkName === newResult.benchmarkName);
79-
if (oldResult) {
80-
const oldValue = oldResult.categories[category];
81-
const newValue = newResult.categories[category];
82-
const percentageChange = calculatePercentageChange(
83-
newResult.categories[category],
84-
oldResult.categories[category]
85-
);
86-
const change = newResult.categories[category] - oldResult.categories[category];
87-
const row = [
88-
newResult.benchmarkName,
89-
formatMemory(oldValue),
90-
formatMemory(newValue),
91-
formatMemory(change),
92-
toFixed(percentageChange, 2),
93-
];
94-
comparisonTable += makeMarkdownTableRow(row);
95-
oldValueSum += oldValue;
96-
newValueSum += newValue;
97-
} else {
98-
// No master found => new benchmark
99-
const row = [newResult.benchmarkName, formatMemory(newResult.categories[category]), "/", "/", "/"];
100-
comparisonTable += makeMarkdownTableRow(row);
101-
}
102-
});
103-
104-
const sumPercentageChange = calculatePercentageChange(oldValueSum, newValueSum);
105-
comparisonTable += makeMarkdownTableRow([
106-
makeBold("sum"),
107-
makeBold(formatMemory(oldValueSum)),
108-
makeBold(formatMemory(newValueSum)),
109-
makeBold(formatMemory(newValueSum - oldValueSum)),
110-
makeBold(toFixed(sumPercentageChange, 2)),
111-
]);
112-
113-
return comparisonTable;
70+
return compareNumericBenchmarks(newResults, oldResults, "mb", result => result.categories[category], formatMemory);
11471
}

0 commit comments

Comments
 (0)