Support for unicode identifiers when targetting LuaJIT (#1187)

tomblind · web-flow · commit 2df70475acd0 · 2021-12-12T20:34:13.000+01:00
implements #1117 Co-authored-by: Tom <tomblind@users.noreply.github.com>
diff --git a/src/LuaPrinter.ts b/src/LuaPrinter.ts
@@ -5,7 +5,7 @@ import * as ts from "typescript";
 import { CompilerOptions, isBundleEnabled, LuaLibImportKind } from "./CompilerOptions";
 import * as lua from "./LuaAST";
 import { loadLuaLibFeatures, LuaLibFeature } from "./LuaLib";
-import { isValidLuaIdentifier } from "./transformation/utils/safe-names";
+import { isValidLuaIdentifier, shouldAllowUnicode } from "./transformation/utils/safe-names";
 import { EmitHost } from "./transpilation";
 import { intersperse, normalizeSlashes } from "./utils";
 
@@ -34,7 +34,8 @@ export const tstlHeader = "--[[ Generated with https://github.com/TypeScriptToLu
  * `foo.bar` => passes (`function foo.bar()` is valid)
  * `getFoo().bar` => fails (`function getFoo().bar()` would be illegal)
  */
-const isValidLuaFunctionDeclarationName = (str: string) => /^[a-zA-Z0-9_.]+$/.test(str);
+const isValidLuaFunctionDeclarationName = (str: string, options: CompilerOptions) =>
+    (shouldAllowUnicode(options) ? /^[a-zA-Z0-9_\u00FF-\uFFFD.]+$/ : /^[a-zA-Z0-9_.]+$/).test(str);
 
 /**
  * Returns true if expression contains no function calls.
@@ -439,7 +440,7 @@ export class LuaPrinter {
         ) {
             // Use `function foo()` instead of `foo = function()`
             const name = this.printExpression(statement.left[0]);
-            if (isValidLuaFunctionDeclarationName(name.toString())) {
+            if (isValidLuaFunctionDeclarationName(name.toString(), this.options)) {
                 chunks.push(this.printFunctionDefinition(statement));
                 return this.createSourceNode(statement, chunks);
             }
@@ -692,7 +693,7 @@ export class LuaPrinter {
         const value = this.printExpression(expression.value);
 
         if (expression.key) {
-            if (lua.isStringLiteral(expression.key) && isValidLuaIdentifier(expression.key.value)) {
+            if (lua.isStringLiteral(expression.key) && isValidLuaIdentifier(expression.key.value, this.options)) {
                 chunks.push(expression.key.value, " = ", value);
             } else {
                 chunks.push("[", this.printExpression(expression.key), "] = ", value);
@@ -804,7 +805,7 @@ export class LuaPrinter {
         const chunks: SourceChunk[] = [];
 
         chunks.push(this.printExpressionInParenthesesIfNeeded(expression.table));
-        if (lua.isStringLiteral(expression.index) && isValidLuaIdentifier(expression.index.value)) {
+        if (lua.isStringLiteral(expression.index) && isValidLuaIdentifier(expression.index.value, this.options)) {
             chunks.push(".", this.createSourceNode(expression.index, expression.index.value));
         } else {
             chunks.push("[", this.printExpression(expression.index), "]");
diff --git a/src/transformation/utils/safe-names.ts b/src/transformation/utils/safe-names.ts
@@ -1,10 +1,19 @@
 import * as ts from "typescript";
+import { CompilerOptions, LuaTarget } from "../..";
 import { TransformationContext } from "../context";
 import { invalidAmbientIdentifierName } from "./diagnostics";
 import { isSymbolExported } from "./export";
 import { isAmbientNode } from "./typescript";
 
-export const isValidLuaIdentifier = (name: string) => !luaKeywords.has(name) && /^[a-zA-Z_][a-zA-Z0-9_]*$/.test(name);
+export const shouldAllowUnicode = (options: CompilerOptions) => options.luaTarget === LuaTarget.LuaJIT;
+
+export const isValidLuaIdentifier = (name: string, options: CompilerOptions) =>
+    !luaKeywords.has(name) &&
+    (shouldAllowUnicode(options)
+        ? /^[a-zA-Z_\u007F-\uFFFD][a-zA-Z0-9_\u007F-\uFFFD]*$/
+        : /^[a-zA-Z_][a-zA-Z0-9_]*$/
+    ).test(name);
+
 export const luaKeywords: ReadonlySet<string> = new Set([
     "and",
     "break",
@@ -52,10 +61,11 @@ const luaBuiltins: ReadonlySet<string> = new Set([
     "unpack",
 ]);
 
-export const isUnsafeName = (name: string) => !isValidLuaIdentifier(name) || luaBuiltins.has(name);
+export const isUnsafeName = (name: string, options: CompilerOptions) =>
+    !isValidLuaIdentifier(name, options) || luaBuiltins.has(name);
 
 function checkName(context: TransformationContext, name: string, node: ts.Node): boolean {
-    const isInvalid = !isValidLuaIdentifier(name);
+    const isInvalid = !isValidLuaIdentifier(name, context.options);
 
     if (isInvalid) {
         // Empty identifier is a TypeScript error
@@ -80,7 +90,7 @@ export function hasUnsafeSymbolName(
     }
 
     // only unsafe when non-ambient and not exported
-    return isUnsafeName(symbol.name) && !isAmbient && !isSymbolExported(context, symbol);
+    return isUnsafeName(symbol.name, context.options) && !isAmbient && !isSymbolExported(context, symbol);
 }
 
 export function hasUnsafeIdentifierName(
diff --git a/src/transformation/visitors/call.ts b/src/transformation/visitors/call.ts
@@ -149,7 +149,7 @@ export function transformContextualCallExpression(
     if (
         ts.isPropertyAccessExpression(left) &&
         ts.isIdentifier(left.name) &&
-        isValidLuaIdentifier(left.name.text) &&
+        isValidLuaIdentifier(left.name.text, context.options) &&
         argPrecedingStatements.length === 0
     ) {
         // table:name()
diff --git a/src/transformation/visitors/class/index.ts b/src/transformation/visitors/class/index.ts
@@ -99,7 +99,7 @@ function transformClassLikeDeclaration(
     const result: lua.Statement[] = [];
 
     let localClassName: lua.Identifier;
-    if (isUnsafeName(className.text)) {
+    if (isUnsafeName(className.text, context.options)) {
         localClassName = lua.createIdentifier(
             createSafeName(className.text),
             undefined,
diff --git a/src/transformation/visitors/namespace.ts b/src/transformation/visitors/namespace.ts
@@ -18,7 +18,7 @@ export function createModuleLocalNameIdentifier(
     declaration: ts.ModuleDeclaration
 ): lua.Identifier {
     const moduleSymbol = context.checker.getSymbolAtLocation(declaration.name);
-    if (moduleSymbol !== undefined && isUnsafeName(moduleSymbol.name)) {
+    if (moduleSymbol !== undefined && isUnsafeName(moduleSymbol.name, context.options)) {
         return lua.createIdentifier(
             createSafeName(declaration.name.text),
             declaration.name,
diff --git a/test/unit/__snapshots__/identifiers.spec.ts.snap b/test/unit/__snapshots__/identifiers.spec.ts.snap
@@ -247,3 +247,183 @@ exports[`undeclared identifier must be a valid lua identifier (object literal sh
 exports[`undeclared identifier must be a valid lua identifier (object literal shorthand) ("ɥɣɎɌͼƛಠ"): code 1`] = `"foo = {[\\"ɥɣɎɌͼƛಠ\\"] = _____265_263_24E_24C_37C_19B_CA0}"`;
 
 exports[`undeclared identifier must be a valid lua identifier (object literal shorthand) ("ɥɣɎɌͼƛಠ"): diagnostics 1`] = `"main.ts(2,27): error TSTL: Invalid ambient identifier name 'ɥɣɎɌͼƛಠ'. Ambient identifiers must be valid lua identifiers."`;
+
+exports[`unicode identifiers in supporting environments (luajit) destructuring property name ("_̀ः٠‿") 1`] = `
+"local ____exports = {}
+function ____exports.__main(self)
+    local ____temp_0 = {foo = \\"foobar\\"}
+    local _̀ः٠‿ = ____temp_0.foo
+    return _̀ः٠‿
+end
+return ____exports"
+`;
+
+exports[`unicode identifiers in supporting environments (luajit) destructuring property name ("ɥɣɎɌͼƛಠ") 1`] = `
+"local ____exports = {}
+function ____exports.__main(self)
+    local ____temp_0 = {foo = \\"foobar\\"}
+    local ɥɣɎɌͼƛಠ = ____temp_0.foo
+    return ɥɣɎɌͼƛಠ
+end
+return ____exports"
+`;
+
+exports[`unicode identifiers in supporting environments (luajit) destructuring property name ("𝛼𝛽𝚫") 1`] = `
+"local ____exports = {}
+function ____exports.__main(self)
+    local ____temp_0 = {foo = \\"foobar\\"}
+    local 𝛼𝛽𝚫 = ____temp_0.foo
+    return 𝛼𝛽𝚫
+end
+return ____exports"
+`;
+
+exports[`unicode identifiers in supporting environments (luajit) destructuring shorthand ("_̀ः٠‿") 1`] = `
+"local ____exports = {}
+function ____exports.__main(self)
+    local ____temp_0 = {_̀ः٠‿ = \\"foobar\\"}
+    local _̀ः٠‿ = ____temp_0._̀ः٠‿
+    return _̀ः٠‿
+end
+return ____exports"
+`;
+
+exports[`unicode identifiers in supporting environments (luajit) destructuring shorthand ("ɥɣɎɌͼƛಠ") 1`] = `
+"local ____exports = {}
+function ____exports.__main(self)
+    local ____temp_0 = {ɥɣɎɌͼƛಠ = \\"foobar\\"}
+    local ɥɣɎɌͼƛಠ = ____temp_0.ɥɣɎɌͼƛಠ
+    return ɥɣɎɌͼƛಠ
+end
+return ____exports"
+`;
+
+exports[`unicode identifiers in supporting environments (luajit) destructuring shorthand ("𝛼𝛽𝚫") 1`] = `
+"local ____exports = {}
+function ____exports.__main(self)
+    local ____temp_0 = {𝛼𝛽𝚫 = \\"foobar\\"}
+    local 𝛼𝛽𝚫 = ____temp_0.𝛼𝛽𝚫
+    return 𝛼𝛽𝚫
+end
+return ____exports"
+`;
+
+exports[`unicode identifiers in supporting environments (luajit) function ("_̀ः٠‿") 1`] = `
+"local ____exports = {}
+function ____exports.__main(self)
+    local function _̀ः٠‿(self, arg)
+        return \\"foo\\" .. arg
+    end
+    return _̀ः٠‿(nil, \\"bar\\")
+end
+return ____exports"
+`;
+
+exports[`unicode identifiers in supporting environments (luajit) function ("ɥɣɎɌͼƛಠ") 1`] = `
+"local ____exports = {}
+function ____exports.__main(self)
+    local function ɥɣɎɌͼƛಠ(self, arg)
+        return \\"foo\\" .. arg
+    end
+    return ɥɣɎɌͼƛಠ(nil, \\"bar\\")
+end
+return ____exports"
+`;
+
+exports[`unicode identifiers in supporting environments (luajit) function ("𝛼𝛽𝚫") 1`] = `
+"local ____exports = {}
+function ____exports.__main(self)
+    local function 𝛼𝛽𝚫(self, arg)
+        return \\"foo\\" .. arg
+    end
+    return 𝛼𝛽𝚫(nil, \\"bar\\")
+end
+return ____exports"
+`;
+
+exports[`unicode identifiers in supporting environments (luajit) identifier name ("_̀ः٠‿") 1`] = `
+"local ____exports = {}
+function ____exports.__main(self)
+    local _̀ः٠‿ = \\"foobar\\"
+    return _̀ः٠‿
+end
+return ____exports"
+`;
+
+exports[`unicode identifiers in supporting environments (luajit) identifier name ("ɥɣɎɌͼƛಠ") 1`] = `
+"local ____exports = {}
+function ____exports.__main(self)
+    local ɥɣɎɌͼƛಠ = \\"foobar\\"
+    return ɥɣɎɌͼƛಠ
+end
+return ____exports"
+`;
+
+exports[`unicode identifiers in supporting environments (luajit) identifier name ("𝛼𝛽𝚫") 1`] = `
+"local ____exports = {}
+function ____exports.__main(self)
+    local 𝛼𝛽𝚫 = \\"foobar\\"
+    return 𝛼𝛽𝚫
+end
+return ____exports"
+`;
+
+exports[`unicode identifiers in supporting environments (luajit) method ("_̀ः٠‿") 1`] = `
+"local ____exports = {}
+function ____exports.__main(self)
+    local foo = {_̀ः٠‿ = function(self, arg)
+        return \\"foo\\" .. arg
+    end}
+    return foo:_̀ः٠‿(\\"bar\\")
+end
+return ____exports"
+`;
+
+exports[`unicode identifiers in supporting environments (luajit) method ("ɥɣɎɌͼƛಠ") 1`] = `
+"local ____exports = {}
+function ____exports.__main(self)
+    local foo = {ɥɣɎɌͼƛಠ = function(self, arg)
+        return \\"foo\\" .. arg
+    end}
+    return foo:ɥɣɎɌͼƛಠ(\\"bar\\")
+end
+return ____exports"
+`;
+
+exports[`unicode identifiers in supporting environments (luajit) method ("𝛼𝛽𝚫") 1`] = `
+"local ____exports = {}
+function ____exports.__main(self)
+    local foo = {𝛼𝛽𝚫 = function(self, arg)
+        return \\"foo\\" .. arg
+    end}
+    return foo:𝛼𝛽𝚫(\\"bar\\")
+end
+return ____exports"
+`;
+
+exports[`unicode identifiers in supporting environments (luajit) property name ("_̀ः٠‿") 1`] = `
+"local ____exports = {}
+function ____exports.__main(self)
+    local x = {_̀ः٠‿ = \\"foobar\\"}
+    return x._̀ः٠‿
+end
+return ____exports"
+`;
+
+exports[`unicode identifiers in supporting environments (luajit) property name ("ɥɣɎɌͼƛಠ") 1`] = `
+"local ____exports = {}
+function ____exports.__main(self)
+    local x = {ɥɣɎɌͼƛಠ = \\"foobar\\"}
+    return x.ɥɣɎɌͼƛಠ
+end
+return ____exports"
+`;
+
+exports[`unicode identifiers in supporting environments (luajit) property name ("𝛼𝛽𝚫") 1`] = `
+"local ____exports = {}
+function ____exports.__main(self)
+    local x = {𝛼𝛽𝚫 = \\"foobar\\"}
+    return x.𝛼𝛽𝚫
+end
+return ____exports"
+`;
diff --git a/test/unit/identifiers.spec.ts b/test/unit/identifiers.spec.ts
@@ -1,3 +1,4 @@
+import { LuaTarget } from "../../src";
 import { invalidAmbientIdentifierName } from "../../src/transformation/utils/diagnostics";
 import { luaKeywords } from "../../src/transformation/utils/safe-names";
 import * as util from "../util";
@@ -222,6 +223,68 @@ test.each(validTsInvalidLuaNames)("exported decorated class with invalid lua nam
         .expectToMatchJsResult();
 });
 
+describe("unicode identifiers in supporting environments (luajit)", () => {
+    const unicodeNames = ["𝛼𝛽𝚫", "ɥɣɎɌͼƛಠ", "_̀ः٠‿"];
+
+    test.each(unicodeNames)("identifier name (%p)", name => {
+        util.testFunction`
+            const ${name} = "foobar";
+            return ${name};
+        `
+            .setOptions({ luaTarget: LuaTarget.LuaJIT })
+            .expectLuaToMatchSnapshot();
+    });
+
+    test.each(unicodeNames)("property name (%p)", name => {
+        util.testFunction`
+            const x = { ${name}: "foobar" };
+            return x.${name};
+        `
+            .setOptions({ luaTarget: LuaTarget.LuaJIT })
+            .expectLuaToMatchSnapshot();
+    });
+
+    test.each(unicodeNames)("destructuring property name (%p)", name => {
+        util.testFunction`
+            const { foo: ${name} } = { foo: "foobar" };
+            return ${name};
+        `
+            .setOptions({ luaTarget: LuaTarget.LuaJIT })
+            .expectLuaToMatchSnapshot();
+    });
+
+    test.each(unicodeNames)("destructuring shorthand (%p)", name => {
+        util.testFunction`
+            const { ${name} } = { ${name}: "foobar" };
+            return ${name};
+        `
+            .setOptions({ luaTarget: LuaTarget.LuaJIT })
+            .expectLuaToMatchSnapshot();
+    });
+
+    test.each(unicodeNames)("function (%p)", name => {
+        util.testFunction`
+            function ${name}(arg: string) {
+                return "foo" + arg;
+            }
+            return ${name}("bar");
+        `
+            .setOptions({ luaTarget: LuaTarget.LuaJIT })
+            .expectLuaToMatchSnapshot();
+    });
+
+    test.each(unicodeNames)("method (%p)", name => {
+        util.testFunction`
+            const foo = {
+                ${name}(arg: string) { return "foo" + arg; }
+            };
+            return foo.${name}("bar");
+        `
+            .setOptions({ luaTarget: LuaTarget.LuaJIT })
+            .expectLuaToMatchSnapshot();
+    });
+});
+
 describe("lua keyword as identifier doesn't interfere with lua's value", () => {
     test("variable (nil)", () => {
         util.testFunction`
