Skip to content

Commit ef69380

Browse files
committed
Added memory benchmarks to CI
1 parent 244633b commit ef69380

File tree

14 files changed

+742
-0
lines changed

14 files changed

+742
-0
lines changed

.github/workflows/ci.yml

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,3 +38,66 @@ jobs:
3838
CI: true
3939
- if: matrix.os == 'ubuntu-latest'
4040
uses: codecov/codecov-action@v1
41+
42+
benchmark:
43+
name: Benchmark
44+
runs-on: ubuntu-latest
45+
steps:
46+
- name: Lua Install
47+
run: sudo apt-get install lua5.3 luajit
48+
- uses: actions/checkout@v2
49+
- name: Use Node.js 12.13.1
50+
uses: actions/setup-node@v1
51+
with:
52+
node-version: 12.13.1
53+
- run: npm ci
54+
- run: npm run build
55+
# This will never result in a direct cache hit.
56+
# Teherefore benchmark data is always updated.
57+
- name: Cache benchmark data
58+
id: cache-benchmark
59+
uses: actions/cache@v1
60+
with:
61+
path: ./benchmark/data
62+
key: ${{ runner.os }}-master-benchmark-${{ github.sha }}
63+
restore-keys: ${{ runner.os }}-master-benchmark-
64+
- name: Ensure benchmark data dir exists
65+
run: mkdir -p ./benchmark/data
66+
- name: Build benchmark Lua 5.3
67+
run: node dist/tstl.js -p benchmark/tsconfig.53.json
68+
- name: Run benchmark Lua 5.3
69+
id: benchmark-lua
70+
run: echo ::set-output name=info::`lua5.3 -- run.lua ../data/benchmark_master_53.json ${{github.ref}}`
71+
working-directory: benchmark/dist
72+
- name: Build benchmark LuaJIT
73+
run: node dist/tstl.js -p benchmark/tsconfig.jit.json
74+
- name: Run benchmark LuaJIT
75+
id: benchmark-jit
76+
run: echo ::set-output name=info::`luajit -- run.lua ../data/benchmark_master_jit.json ${{github.ref}}`
77+
working-directory: benchmark/dist
78+
- name: Create benchmark check
79+
uses: actions/github-script@0.9.0
80+
with:
81+
benchmark-info-lua: ${{steps.benchmark-lua.outputs.info}}
82+
benchmark-info-jit: ${{steps.benchmark-jit.outputs.info}}
83+
script: |
84+
const benchmarkInfoLua = JSON.parse(core.getInput('benchmark-info-lua', { required: true }));
85+
const benchmarkInfoJIT = JSON.parse(core.getInput('benchmark-info-jit', { required: true }));
86+
87+
const summary = `### Lua5.3\n${benchmarkInfoLua.summary}\n### LuaJIT\n${benchmarkInfoJIT.summary}`;
88+
89+
const text = `### Lua5.3\n${benchmarkInfoLua.text}\n### LuaJIT\n${benchmarkInfoJIT.text}`;
90+
91+
github.checks.create({
92+
owner: context.repo.owner,
93+
repo: context.repo.repo,
94+
name: "Benchmark results",
95+
head_sha: context.sha,
96+
status: "completed",
97+
conclusion: "neutral",
98+
output: {
99+
title: "Benchmark results",
100+
summary: summary,
101+
text: text
102+
}
103+
});

.gitignore

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,3 +6,7 @@ yarn.lock
66
.vscode
77
.idea
88
.DS_Store
9+
10+
benchmark/data/*
11+
benchmark/dist/*
12+
!benchmark/dist/json.lua

benchmark/README.md

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
## TSTL Benchmarks
2+
3+
These benchmarks are written in typescript and transpiled to lua by using tstl.
4+
5+
### Currently only memory benchmarks are supported
6+
7+
To add a new benchmark add a new file to `memory_benchmarks`
8+
with a exported function with the following type: `() => void`.
9+
10+
And add the function to the `memoryBenchmarkInput` inside `run.ts`.
11+
12+
For example:
13+
14+
```ts
15+
export default myBenchmark() {
16+
cont n = 123;
17+
for (let i = 0; i < n; i++) {
18+
// Do something memory instensive
19+
}
20+
}
21+
```
22+
23+
```ts
24+
import myBenchmark from "./memory_benchmarks/myBenchmark";
25+
26+
// ...
27+
28+
const memoryBenchmarkInput: (() => void)[] = [
29+
// ...
30+
myBenchmark
31+
];
32+
```
33+
34+
**Goal**
35+
36+
The goal of memory benchmarks is to track how much (memory) `"garbage"` is created by tstl.
37+
For that reason garabage collection is disabled in the benchmarks.
38+
39+
You can force the creation of `"garbage"` by creating a lot of anonymous functions or temporary tables (see [lua-users.org](http://lua-users.org/wiki/OptimisingGarbageCollection) for more information).
40+
41+
To avoid crashes in the CI your benchmark should not use more than 500MB of memory.
42+
43+
**Running locally**
44+
45+
`npx typescript-to-lua -p tsconfig.53.json && cd dist && lua -- run.lua ../data/benchmark_master_53.json master`

benchmark/benchmark_types.ts

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
export enum BenchmarkKind {
2+
Memory = "memory",
3+
}
4+
5+
export interface BenchmarkResult {
6+
kind: BenchmarkKind
7+
}
8+
9+
export interface MemoryBenchmarkResult extends BenchmarkResult {
10+
kind: BenchmarkKind.Memory
11+
benchmarkName: string;
12+
preExecMemoryUsage: number,
13+
postExecMemoryUsage: number,
14+
memoryUsedForExec: number,
15+
memoryAfterGC: number,
16+
}
17+
18+
export function isMemoryBenchmarkResult(result: BenchmarkResult): result is MemoryBenchmarkResult {
19+
return result.kind == BenchmarkKind.Memory;
20+
}

0 commit comments

Comments
 (0)