Skip to content

Commit 404ab4e

Browse files
ark120202ark120202
authored
Refactor transpilation pipeline (#885)
* Refactor transpilation pipeline * Fix bundle main file finding * Fix transpile tests * Fix source map tests * Add comment to inline source maps test * Rename `transpile` to `getProgramTranspileResult` and export it * Move emit output collector to it's own file * Add missing TranspiledFile reexport
1 parent 0d7c5cd commit 404ab4e

21 files changed

+364
-374
lines changed

‎build-lualib.js‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,8 @@ const tstl = require("./src");
66
const { loadLuaLibFeatures } = require("./src/LuaLib");
77

88
const configFileName = path.resolve(__dirname, "src/lualib/tsconfig.json");
9-
const { emitResult, diagnostics } = tstl.transpileProject(configFileName);
10-
emitResult.forEach(({ name, text }) => ts.sys.writeFile(name, text));
11-
12-
const reportDiagnostic = tstl.createDiagnosticReporter(true);
13-
diagnostics.forEach(reportDiagnostic);
9+
const { diagnostics } = tstl.transpileProject(configFileName);
10+
diagnostics.forEach(tstl.createDiagnosticReporter(true));
1411

1512
const bundlePath = path.join(__dirname, "dist/lualib/lualib_bundle.lua");
1613
if (fs.existsSync(bundlePath)) {

‎src/CompilerOptions.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,9 @@ export enum LuaTarget {
5151
LuaJIT = "JIT",
5252
}
5353

54+
export const isBundleEnabled = (options: CompilerOptions) =>
55+
options.luaBundle !== undefined && options.luaBundleEntry !== undefined;
56+
5457
export function validateOptions(options: CompilerOptions): ts.Diagnostic[] {
5558
const diagnostics: ts.Diagnostic[] = [];
5659

‎src/transpilation/bundle.ts‎

Lines changed: 44 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -2,101 +2,90 @@ import * as path from "path";
22
import { SourceNode } from "source-map";
33
import * as ts from "typescript";
44
import { CompilerOptions } from "../CompilerOptions";
5-
import { getLuaLibBundle } from "../LuaLib";
65
import { escapeString } from "../LuaPrinter";
7-
import { formatPathToLuaPath, normalizeSlashes, trimExtension } from "../utils";
8-
import * as diagnosticFactories from "./diagnostics";
9-
import { EmitHost, TranspiledFile } from "./transpile";
6+
import { cast, formatPathToLuaPath, isNonNull, normalizeSlashes, trimExtension } from "../utils";
7+
import { couldNotFindBundleEntryPoint } from "./diagnostics";
8+
import { EmitFile, EmitHost, ProcessedFile } from "./utils";
109

1110
const createModulePath = (baseDir: string, pathToResolve: string) =>
1211
escapeString(formatPathToLuaPath(trimExtension(path.relative(baseDir, pathToResolve))));
1312

14-
export function bundleTranspiledFiles(
15-
bundleFile: string,
16-
entryModule: string,
17-
transpiledFiles: TranspiledFile[],
13+
// Override `require` to read from ____modules table.
14+
const requireOverride = `
15+
local ____modules = {}
16+
local ____moduleCache = {}
17+
local ____originalRequire = require
18+
local function require(file)
19+
if ____moduleCache[file] then
20+
return ____moduleCache[file]
21+
end
22+
if ____modules[file] then
23+
____moduleCache[file] = ____modules[file]()
24+
return ____moduleCache[file]
25+
else
26+
if ____originalRequire then
27+
return ____originalRequire(file)
28+
else
29+
error("module '" .. file .. "' not found")
30+
end
31+
end
32+
end
33+
`;
34+
35+
export function getBundleResult(
1836
program: ts.Program,
19-
emitHost: EmitHost
20-
): [ts.Diagnostic[], TranspiledFile] {
37+
emitHost: EmitHost,
38+
files: ProcessedFile[]
39+
): [ts.Diagnostic[], EmitFile] {
2140
const diagnostics: ts.Diagnostic[] = [];
2241

2342
const options = program.getCompilerOptions() as CompilerOptions;
43+
const bundleFile = cast(options.luaBundle, isNonNull);
44+
const entryModule = cast(options.luaBundleEntry, isNonNull);
2445

46+
const rootDir = program.getCommonSourceDirectory();
47+
const outDir = options.outDir ?? rootDir;
2548
const projectRootDir = options.configFilePath
2649
? path.dirname(options.configFilePath)
2750
: emitHost.getCurrentDirectory();
2851

2952
// Resolve project settings relative to project file.
3053
const resolvedEntryModule = path.resolve(projectRootDir, entryModule);
31-
const resolvedBundleFile = path.resolve(projectRootDir, bundleFile);
54+
const outputPath = normalizeSlashes(path.resolve(projectRootDir, bundleFile));
3255

33-
// Resolve source files relative to common source directory.
34-
const sourceRootDir = program.getCommonSourceDirectory();
35-
if (!transpiledFiles.some(f => path.resolve(sourceRootDir, f.fileName) === resolvedEntryModule)) {
36-
return [[diagnosticFactories.couldNotFindBundleEntryPoint(entryModule)], { fileName: bundleFile }];
56+
if (!files.some(f => f.fileName === resolvedEntryModule)) {
57+
diagnostics.push(couldNotFindBundleEntryPoint(entryModule));
58+
return [diagnostics, { outputPath, code: "" }];
3759
}
3860

3961
// For each file: ["<module path>"] = function() <lua content> end,
40-
const moduleTableEntries: SourceChunk[] = transpiledFiles.map(f =>
41-
moduleSourceNode(f, createModulePath(sourceRootDir, f.fileName))
42-
);
43-
44-
// If any of the modules contains a require for lualib_bundle, add it to the module table.
45-
const lualibRequired = transpiledFiles.some(f => f.lua?.includes('require("lualib_bundle")'));
46-
if (lualibRequired) {
47-
moduleTableEntries.push(`["lualib_bundle"] = function() ${getLuaLibBundle(emitHost)} end,\n`);
48-
}
62+
const moduleTableEntries = files.map(f => moduleSourceNode(f, createModulePath(outDir, f.fileName)));
4963

5064
// Create ____modules table containing all entries from moduleTableEntries
5165
const moduleTable = createModuleTableNode(moduleTableEntries);
5266

53-
// Override `require` to read from ____modules table.
54-
const requireOverride = `
55-
local ____modules = {}
56-
local ____moduleCache = {}
57-
local ____originalRequire = require
58-
local function require(file)
59-
if ____moduleCache[file] then
60-
return ____moduleCache[file]
61-
end
62-
if ____modules[file] then
63-
____moduleCache[file] = ____modules[file]()
64-
return ____moduleCache[file]
65-
else
66-
if ____originalRequire then
67-
return ____originalRequire(file)
68-
else
69-
error("module '" .. file .. "' not found")
70-
end
71-
end
72-
end\n`;
73-
7467
// return require("<entry module path>")
75-
const entryPoint = `return require(${createModulePath(sourceRootDir, resolvedEntryModule)})\n`;
68+
const entryPoint = `return require(${createModulePath(outDir, resolvedEntryModule)})\n`;
7669

7770
const bundleNode = joinSourceChunks([requireOverride, moduleTable, entryPoint]);
7871
const { code, map } = bundleNode.toStringWithSourceMap();
7972

8073
return [
8174
diagnostics,
8275
{
83-
fileName: normalizeSlashes(resolvedBundleFile),
84-
lua: code,
76+
outputPath,
77+
code,
8578
sourceMap: map.toString(),
86-
sourceMapNode: moduleTable,
79+
sourceFiles: files.flatMap(x => x.sourceFiles ?? []),
8780
},
8881
];
8982
}
9083

91-
function moduleSourceNode(transpiledFile: TranspiledFile, modulePath: string): SourceNode {
84+
function moduleSourceNode({ code, sourceMapNode }: ProcessedFile, modulePath: string): SourceNode {
9285
const tableEntryHead = `[${modulePath}] = function() `;
9386
const tableEntryTail = "end,\n";
9487

95-
if (transpiledFile.lua && transpiledFile.sourceMapNode) {
96-
return joinSourceChunks([tableEntryHead, transpiledFile.sourceMapNode, tableEntryTail]);
97-
} else {
98-
return joinSourceChunks([tableEntryHead, tableEntryTail]);
99-
}
88+
return joinSourceChunks([tableEntryHead, sourceMapNode ?? code, tableEntryTail]);
10089
}
10190

10291
function createModuleTableNode(fileChunks: SourceChunk[]): SourceNode {

‎src/transpilation/emit.ts‎

Lines changed: 0 additions & 69 deletions
This file was deleted.

‎src/transpilation/index.ts‎

Lines changed: 48 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -3,38 +3,41 @@ import * as path from "path";
33
import * as ts from "typescript";
44
import { parseConfigFileWithSystem } from "../cli/tsconfig";
55
import { CompilerOptions } from "../CompilerOptions";
6-
import { emitTranspiledFiles, OutputFile } from "./emit";
7-
import { transpile, TranspiledFile, TranspileResult } from "./transpile";
6+
import { createEmitOutputCollector, TranspiledFile } from "./output-collector";
7+
import { EmitResult, Transpiler } from "./transpiler";
88

9-
export * from "./emit";
109
export { Plugin } from "./plugins";
1110
export * from "./transpile";
12-
13-
export interface TranspileFilesResult {
14-
diagnostics: ts.Diagnostic[];
15-
emitResult: OutputFile[];
16-
}
17-
18-
export function transpileFiles(rootNames: string[], options: CompilerOptions = {}): TranspileFilesResult {
11+
export * from "./transpiler";
12+
export { EmitHost } from "./utils";
13+
export { TranspiledFile };
14+
15+
export function transpileFiles(
16+
rootNames: string[],
17+
options: CompilerOptions = {},
18+
writeFile?: ts.WriteFileCallback
19+
): EmitResult {
1920
const program = ts.createProgram(rootNames, options);
20-
const { transpiledFiles, diagnostics: transpileDiagnostics } = transpile({ program });
21-
const emitResult = emitTranspiledFiles(program, transpiledFiles);
22-
21+
const { diagnostics: transpileDiagnostics, emitSkipped } = new Transpiler().emit({ program, writeFile });
2322
const diagnostics = ts.sortAndDeduplicateDiagnostics([
2423
...ts.getPreEmitDiagnostics(program),
2524
...transpileDiagnostics,
2625
]);
2726

28-
return { diagnostics: [...diagnostics], emitResult };
27+
return { diagnostics: [...diagnostics], emitSkipped };
2928
}
3029

31-
export function transpileProject(configFileName: string, optionsToExtend?: CompilerOptions): TranspileFilesResult {
30+
export function transpileProject(
31+
configFileName: string,
32+
optionsToExtend?: CompilerOptions,
33+
writeFile?: ts.WriteFileCallback
34+
): EmitResult {
3235
const parseResult = parseConfigFileWithSystem(configFileName, optionsToExtend);
3336
if (parseResult.errors.length > 0) {
34-
return { diagnostics: parseResult.errors, emitResult: [] };
37+
return { diagnostics: parseResult.errors, emitSkipped: true };
3538
}
3639

37-
return transpileFiles(parseResult.fileNames, parseResult.options);
40+
return transpileFiles(parseResult.fileNames, parseResult.options, writeFile);
3841
}
3942

4043
const libCache: { [key: string]: ts.SourceFile } = {};
@@ -51,33 +54,45 @@ export function createVirtualProgram(input: Record<string, string>, options: Com
5154
useCaseSensitiveFileNames: () => false,
5255
writeFile() {},
5356

54-
getSourceFile(filename) {
55-
if (filename in input) {
56-
return ts.createSourceFile(filename, input[filename], ts.ScriptTarget.Latest, false);
57+
getSourceFile(fileName) {
58+
if (fileName in input) {
59+
return ts.createSourceFile(fileName, input[fileName], ts.ScriptTarget.Latest, false);
5760
}
5861

59-
if (filename.startsWith("lib.")) {
60-
if (libCache[filename]) return libCache[filename];
62+
if (fileName.startsWith("lib.")) {
63+
if (libCache[fileName]) return libCache[fileName];
6164
const typeScriptDir = path.dirname(require.resolve("typescript"));
62-
const filePath = path.join(typeScriptDir, filename);
65+
const filePath = path.join(typeScriptDir, fileName);
6366
const content = fs.readFileSync(filePath, "utf8");
6467

65-
libCache[filename] = ts.createSourceFile(filename, content, ts.ScriptTarget.Latest, false);
68+
libCache[fileName] = ts.createSourceFile(fileName, content, ts.ScriptTarget.Latest, false);
6669

67-
return libCache[filename];
70+
return libCache[fileName];
6871
}
6972
},
7073
};
7174

7275
return ts.createProgram(Object.keys(input), options, compilerHost);
7376
}
7477

75-
export function transpileVirtualProject(files: Record<string, string>, options: CompilerOptions = {}): TranspileResult {
78+
export interface TranspileVirtualProjectResult {
79+
diagnostics: ts.Diagnostic[];
80+
transpiledFiles: TranspiledFile[];
81+
}
82+
83+
export function transpileVirtualProject(
84+
files: Record<string, string>,
85+
options: CompilerOptions = {}
86+
): TranspileVirtualProjectResult {
7687
const program = createVirtualProgram(files, options);
77-
const result = transpile({ program });
78-
const diagnostics = ts.sortAndDeduplicateDiagnostics([...ts.getPreEmitDiagnostics(program), ...result.diagnostics]);
88+
const collector = createEmitOutputCollector();
89+
const { diagnostics: transpileDiagnostics } = new Transpiler().emit({ program, writeFile: collector.writeFile });
90+
const diagnostics = ts.sortAndDeduplicateDiagnostics([
91+
...ts.getPreEmitDiagnostics(program),
92+
...transpileDiagnostics,
93+
]);
7994

80-
return { ...result, diagnostics: [...diagnostics] };
95+
return { diagnostics: [...diagnostics], transpiledFiles: collector.files };
8196
}
8297

8398
export interface TranspileStringResult {
@@ -87,5 +102,8 @@ export interface TranspileStringResult {
87102

88103
export function transpileString(main: string, options: CompilerOptions = {}): TranspileStringResult {
89104
const { diagnostics, transpiledFiles } = transpileVirtualProject({ "main.ts": main }, options);
90-
return { diagnostics, file: transpiledFiles.find(({ fileName }) => fileName === "main.ts") };
105+
return {
106+
diagnostics,
107+
file: transpiledFiles.find(({ sourceFiles }) => sourceFiles.some(f => f.fileName === "main.ts")),
108+
};
91109
}

0 commit comments

Comments
 (0)