TypeScriptToLua · Perryvw · May 11, 2020 · May 5, 2020 · May 5, 2020 · May 5, 2020
diff --git a/.eslintrc.js b/.eslintrc.js
@@ -67,7 +67,10 @@ const typescriptBase = {
 
 module.exports = {
     extends: ["plugin:jest/recommended", "plugin:jest/style"],
-    parserOptions: { sourceType: "module", project: ["test/tsconfig.json", "src/lualib/tsconfig.json"] },
+    parserOptions: {
+        sourceType: "module",
+        project: ["test/tsconfig.json", "src/lualib/tsconfig.json", "benchmark/tsconfig.json"],
+    },
     env: { es6: true, node: true },
     plugins: ["import"],
     rules: {
@@ -186,5 +189,11 @@ module.exports = {
                 "@typescript-eslint/prefer-optional-chain": "off",
             },
         },
+        {
+            files: "benchmark/src/memory_benchmarks/**/*.ts",
+            rules: {
+                "import/no-default-export": "off",
+            },
+        },
     ],
 };
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -38,3 +38,94 @@ jobs:
           CI: true
       - if: matrix.os == 'ubuntu-latest'
         uses: codecov/codecov-action@v1
+
+  benchmark:
+    name: Benchmark
+    runs-on: ubuntu-latest
+    steps:
+      - name: Lua Install
+        run: sudo apt-get install lua5.3 luajit
+      # Checkout master & commit
+      - name: Checkout master
+        uses: actions/checkout@v2
+        with:
+          ref: master
+          path: master
+      - name: Checkout commit
+        uses: actions/checkout@v2
+        with:
+          path: commit
+      - name: Use Node.js 12.13.1
+        uses: actions/setup-node@v1
+        with:
+          node-version: 12.13.1
+      # NPM
+      - name: NPM master
+        # TODO Lua types is only added manually to test the benchmark PR this can be removed again once the PR is merged
+        run: npm ci && npm run build && npm install -D lua-types
+        working-directory: master
+      - name: NPM commit
+        run: npm ci && npm run build
+        working-directory: commit
+      # Benchmark directory setup
+      - name: Ensure benchmark data dir exists
+        run: mkdir -p ./benchmark/data
+        working-directory: commit
+      - name: Copy commit benchmark to master
+        run: rm -rf ./master/benchmark && cp -rf ./commit/benchmark ./master/benchmark
+      # Run master benchmark first and output to commit benchmark data
+      - name: Build benchmark Lua 5.3 master
+        run: node ../dist/tstl.js -p tsconfig.53.json
+        working-directory: master/benchmark
+      - name: Run benchmark Lua 5.3 master
+        id: benchmark-lua-master
+        run: lua5.3 -- run.lua ../../../commit/benchmark/data/benchmark_master_53.json
+        working-directory: master/benchmark/dist
+      - name: Build benchmark LuaJIT master
+        run: node ../dist/tstl.js -p tsconfig.jit.json
+        working-directory: master/benchmark
+      - name: Run benchmark LuaJIT master
+        id: benchmark-jit-master
+        run: luajit -- run.lua ../../../commit/benchmark/data/benchmark_master_jit.json
+        working-directory: master/benchmark/dist
+      # Run commit benchmark and compare with master
+      - name: Build benchmark Lua 5.3 commit
+        run: node ../dist/tstl.js -p tsconfig.53.json
+        working-directory: commit/benchmark
+      - name: Run benchmark Lua 5.3 commit
+        id: benchmark-lua-commit
+        run: echo ::set-output name=info::`lua5.3 -- run.lua ../data/benchmark_commit_53.json ../data/benchmark_master_53.json`
+        working-directory: commit/benchmark/dist
+      - name: Build benchmark LuaJIT commit
+        run: node ../dist/tstl.js -p tsconfig.jit.json
+        working-directory: commit/benchmark
+      - name: Run benchmark LuaJIT commit
+        id: benchmark-jit-commit
+        run: echo ::set-output name=info::`luajit -- run.lua ../data/benchmark_commit_jit.json ../data/benchmark_master_jit.json`
+        working-directory: commit/benchmark/dist
+      - name: Create benchmark check
+        uses: actions/github-script@0.9.0
+        with:
+          benchmark-info-lua: ${{steps.benchmark-lua-commit.outputs.info}}
+          benchmark-info-jit: ${{steps.benchmark-jit-commit.outputs.info}}
+          script: |
+            const benchmarkInfoLua = JSON.parse(core.getInput('benchmark-info-lua', { required: true }));
+            const benchmarkInfoJIT = JSON.parse(core.getInput('benchmark-info-jit', { required: true }));
+
+            const summary = `### Lua5.3\n${benchmarkInfoLua.summary}\n### LuaJIT\n${benchmarkInfoJIT.summary}`;
+
+            const text = `### Lua5.3\n${benchmarkInfoLua.text}\n### LuaJIT\n${benchmarkInfoJIT.text}`;
+
+            github.checks.create({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              name: "Benchmark results",
+              head_sha: context.sha,
+              status: "completed",
+              conclusion: "neutral",
+              output: {
+                title: "Benchmark results",
+                summary: summary,
+                text: text
+              }
+            });
diff --git a/.gitignore b/.gitignore
@@ -6,3 +6,7 @@ yarn.lock
 .vscode
 .idea
 .DS_Store
+
+benchmark/data/*
+benchmark/dist/*
+!benchmark/dist/json.lua
diff --git a/benchmark/README.md b/benchmark/README.md
@@ -0,0 +1,43 @@
+## TSTL Benchmarks
+
+These benchmarks are written in typescript and transpiled to lua by using tstl.
+
+### Currently only memory benchmarks are supported
+
+To add a new benchmark add a new file to `memory_benchmarks`
+and **default** export a function with the following type: `() => void`.
+To prevent the benchmark from reporting "useful" results of your benchmark function as garbage, simply return the result.
+The memory used by the returned result wont count towards the total garbage amount.
+
+For example (memory_benchmarks/my_benchmark.ts):
+
+```ts
+export default function myBenchmark() {
+  const n = 123;
+  const result = [];
+  for (let i = 0; i < n; i++) {
+    // Do something memory instensive
+  }
+  return result; // Return results so they wont be counted as garbage
+}
+```
+
+**Goal**
+
+The goal of memory benchmarks is to track how much (memory) `"garbage"` is created by tstl.
+For that reason garabage collection is disabled in the benchmarks.
+
+You can force the creation of `"garbage"` by creating a lot of anonymous functions or temporary tables (see [lua-users.org](http://lua-users.org/wiki/OptimisingGarbageCollection) for more information).
+
+To avoid crashes in the CI your benchmark should not use more than 500MB of memory.
+
+**Running locally**
+
+1. Create a benchmark baseline called "benchmark_baseline.json":  
+   `tstl -p tsconfig.53.json && cd dist && lua -- run.lua benchmark_baseline.json`
+2. Make some changes to tstl.
+3. Create an updated benchmark and compare with the baseline:  
+   `tstl -p tsconfig.53.json && cd dist && lua -- run.lua benchmark_updated.json benchmark_baseline.json`
+4. The above command will output comparison data as json to stdout.
+   If you provide a path as third argument the comparison data will be written to that path instead.  
+   `tstl -p tsconfig.53.json && cd dist && lua -- run.lua benchmark_updated.json benchmark_baseline.json result.md`