Skip to content

Commit ab79fae

Browse files
DanielRosenwasserDanielRosenwasser
committed
Added tests, fixed order of emptying templateStack, unconditionally perform template classification.
1 parent 3fea0ae commit ab79fae

2 files changed

Lines changed: 137 additions & 36 deletions

File tree

‎src/services/services.ts‎

Lines changed: 18 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -5663,6 +5663,11 @@ module ts {
56635663
var token = SyntaxKind.Unknown;
56645664
var lastNonTriviaToken = SyntaxKind.Unknown;
56655665

5666+
// Empty out the template stack for reuse.
5667+
while (templateStack.length > 0) {
5668+
templateStack.pop();
5669+
}
5670+
56665671
// If we're in a string literal, then prepend: "\
56675672
// (and a newline). That way when we lex we'll think we're still in a string literal.
56685673
//
@@ -5682,21 +5687,15 @@ module ts {
56825687
offset = 3;
56835688
break;
56845689
case EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate:
5685-
if (syntacticClassifierAbsent) {
5686-
text = "`\n" + text;
5687-
offset = 2;
5688-
}
5690+
text = "`\n" + text;
5691+
offset = 2;
56895692
break;
56905693
case EndOfLineState.InTemplateMiddleOrTail:
5691-
if (syntacticClassifierAbsent) {
5692-
text = "}\n" + text;
5693-
offset = 2;
5694-
}
5694+
text = "}\n" + text;
5695+
offset = 2;
56955696
// fallthrough
56965697
case EndOfLineState.InTemplateSubstitutionPosition:
5697-
if (syntacticClassifierAbsent) {
5698-
templateStack = [SyntaxKind.TemplateHead];
5699-
}
5698+
templateStack.push(SyntaxKind.TemplateHead);
57005699
break;
57015700
}
57025701

@@ -5728,11 +5727,6 @@ module ts {
57285727
// work well enough in practice.
57295728
var angleBracketStack = 0;
57305729

5731-
// Empty out the template stack for reuse.
5732-
while (templateStack.length > 0) {
5733-
templateStack.pop();
5734-
}
5735-
57365730
do {
57375731
token = scanner.scan();
57385732

@@ -5774,17 +5768,17 @@ module ts {
57745768
token = SyntaxKind.Identifier;
57755769
}
57765770
}
5777-
else if (token === SyntaxKind.TemplateHead && syntacticClassifierAbsent) {
5771+
else if (token === SyntaxKind.TemplateHead) {
57785772
templateStack.push(token);
57795773
}
5780-
else if (token === SyntaxKind.OpenBraceToken && syntacticClassifierAbsent) {
5774+
else if (token === SyntaxKind.OpenBraceToken) {
57815775
// If we don't have anything on the template stack,
57825776
// then we aren't trying to keep track of a previously scanned template head.
57835777
if (templateStack.length > 0) {
57845778
templateStack.push(token);
57855779
}
57865780
}
5787-
else if (token === SyntaxKind.CloseBraceToken && syntacticClassifierAbsent) {
5781+
else if (token === SyntaxKind.CloseBraceToken) {
57885782
// If we don't have anything on the template stack,
57895783
// then we aren't trying to keep track of a previously scanned template head.
57905784
if (templateStack.length > 0) {
@@ -5821,7 +5815,7 @@ module ts {
58215815
var start = scanner.getTokenPos();
58225816
var end = scanner.getTextPos();
58235817

5824-
addResult(end - start, classFromKind(token, syntacticClassifierAbsent));
5818+
addResult(end - start, classFromKind(token));
58255819

58265820
if (end >= text.length) {
58275821
if (token === SyntaxKind.StringLiteral) {
@@ -5850,7 +5844,7 @@ module ts {
58505844
result.finalLexState = EndOfLineState.InMultiLineCommentTrivia;
58515845
}
58525846
}
5853-
else if (isTemplateLiteralKind(token) && syntacticClassifierAbsent) {
5847+
else if (isTemplateLiteralKind(token)) {
58545848
if (scanner.isUnterminated()) {
58555849
if (token === SyntaxKind.TemplateTail) {
58565850
result.finalLexState = EndOfLineState.InTemplateMiddleOrTail;
@@ -5943,7 +5937,7 @@ module ts {
59435937
return token >= SyntaxKind.FirstKeyword && token <= SyntaxKind.LastKeyword;
59445938
}
59455939

5946-
function classFromKind(token: SyntaxKind, syntacticClassifierAbsent?: boolean) {
5940+
function classFromKind(token: SyntaxKind) {
59475941
if (isKeyword(token)) {
59485942
return TokenClass.Keyword;
59495943
}
@@ -5969,9 +5963,8 @@ module ts {
59695963
return TokenClass.Whitespace;
59705964
case SyntaxKind.Identifier:
59715965
default:
5972-
// Only give a classification if nothing will more accurately classify.
5973-
if (syntacticClassifierAbsent && isTemplateLiteralKind(token)) {
5974-
return TokenClass.StringLiteral; // should make a TemplateLiteral
5966+
if (isTemplateLiteralKind(token)) {
5967+
return TokenClass.StringLiteral; // maybe make a TemplateLiteral
59755968
}
59765969
return TokenClass.Identifier;
59775970
}

‎tests/cases/unittests/services/colorization.ts‎

Lines changed: 119 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
interface ClassificationEntry {
55
value: any;
66
classification: ts.TokenClass;
7+
position?: number;
78
}
89

910
describe('Colorization', function () {
@@ -23,16 +24,23 @@ describe('Colorization', function () {
2324
return undefined;
2425
}
2526

26-
function punctuation(text: string): ClassificationEntry { return { value: text, classification: ts.TokenClass.Punctuation }; }
27-
function keyword(text: string): ClassificationEntry { return { value: text, classification: ts.TokenClass.Keyword }; }
28-
function operator(text: string): ClassificationEntry { return { value: text, classification: ts.TokenClass.Operator }; }
29-
function comment(text: string): ClassificationEntry { return { value: text, classification: ts.TokenClass.Comment }; }
30-
function whitespace(text: string): ClassificationEntry { return { value: text, classification: ts.TokenClass.Whitespace }; }
31-
function identifier(text: string): ClassificationEntry { return { value: text, classification: ts.TokenClass.Identifier }; }
32-
function numberLiteral(text: string): ClassificationEntry { return { value: text, classification: ts.TokenClass.NumberLiteral }; }
33-
function stringLiteral(text: string): ClassificationEntry { return { value: text, classification: ts.TokenClass.StringLiteral }; }
34-
function regExpLiteral(text: string): ClassificationEntry { return { value: text, classification: ts.TokenClass.RegExpLiteral }; }
35-
function finalEndOfLineState(value: number): ClassificationEntry { return { value: value, classification: <ts.TokenClass>undefined }; }
27+
function punctuation(text: string, position?: number) { return createClassification(text, ts.TokenClass.Punctuation, position); }
28+
function keyword(text: string, position?: number) { return createClassification(text, ts.TokenClass.Keyword, position); }
29+
function operator(text: string, position?: number) { return createClassification(text, ts.TokenClass.Operator, position); }
30+
function comment(text: string, position?: number) { return createClassification(text, ts.TokenClass.Comment, position); }
31+
function whitespace(text: string, position?: number) { return createClassification(text, ts.TokenClass.Whitespace, position); }
32+
function identifier(text: string, position?: number) { return createClassification(text, ts.TokenClass.Identifier, position); }
33+
function numberLiteral(text: string, position?: number) { return createClassification(text, ts.TokenClass.NumberLiteral, position); }
34+
function stringLiteral(text: string, position?: number) { return createClassification(text, ts.TokenClass.StringLiteral, position); }
35+
function regExpLiteral(text: string, position?: number) { return createClassification(text, ts.TokenClass.RegExpLiteral, position); }
36+
function finalEndOfLineState(value: number): ClassificationEntry { return { value: value, classification: undefined, position: 0 }; }
37+
function createClassification(text: string, tokenClass: ts.TokenClass, position?: number): ClassificationEntry {
38+
return {
39+
value: text,
40+
classification: tokenClass,
41+
position: position,
42+
};
43+
}
3644

3745
function testLexicalClassification(text: string, initialEndOfLineState: ts.EndOfLineState, ...expectedEntries: ClassificationEntry[]): void {
3846
var result = classifier.getClassificationsForLine(text, initialEndOfLineState);
@@ -44,7 +52,7 @@ describe('Colorization', function () {
4452
assert.equal(result.finalLexState, expectedEntry.value, "final endOfLineState does not match expected.");
4553
}
4654
else {
47-
var actualEntryPosition = text.indexOf(expectedEntry.value);
55+
var actualEntryPosition = expectedEntry.position !== undefined ? expectedEntry.position : text.indexOf(expectedEntry.value);
4856
assert(actualEntryPosition >= 0, "token: '" + expectedEntry.value + "' does not exit in text: '" + text + "'.");
4957

5058
var actualEntry = getEntryAtPosistion(result, actualEntryPosition);
@@ -254,6 +262,106 @@ describe('Colorization', function () {
254262
finalEndOfLineState(ts.EndOfLineState.Start));
255263
});
256264

265+
it("classifies a single line no substitution template string correctly", () => {
266+
testLexicalClassification("`number number public string`",
267+
ts.EndOfLineState.Start,
268+
stringLiteral("`number number public string`"),
269+
finalEndOfLineState(ts.EndOfLineState.Start));
270+
});
271+
it("classifies substitution parts of a template string correctly", () => {
272+
testLexicalClassification("`number '${ 1 + 1 }' string '${ 'hello' }'`",
273+
ts.EndOfLineState.Start,
274+
stringLiteral("`number '${"),
275+
numberLiteral("1"),
276+
operator("+"),
277+
numberLiteral("1"),
278+
stringLiteral("}' string '${"),
279+
stringLiteral("'hello'"),
280+
stringLiteral("}'`"),
281+
finalEndOfLineState(ts.EndOfLineState.Start));
282+
});
283+
it("classifies an unterminated no substitution template string correctly", () => {
284+
testLexicalClassification("`hello world",
285+
ts.EndOfLineState.Start,
286+
stringLiteral("`hello world"),
287+
finalEndOfLineState(ts.EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate));
288+
});
289+
it("classifies the entire line of an unterminated multiline no-substitution/head template", () => {
290+
testLexicalClassification("...",
291+
ts.EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate,
292+
stringLiteral("..."),
293+
finalEndOfLineState(ts.EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate));
294+
});
295+
it("classifies the entire line of an unterminated multiline template middle/end",() => {
296+
testLexicalClassification("...",
297+
ts.EndOfLineState.InTemplateMiddleOrTail,
298+
stringLiteral("..."),
299+
finalEndOfLineState(ts.EndOfLineState.InTemplateMiddleOrTail));
300+
});
301+
it("classifies a termination of a multiline template head", () => {
302+
testLexicalClassification("...${",
303+
ts.EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate,
304+
stringLiteral("...${"),
305+
finalEndOfLineState(ts.EndOfLineState.InTemplateSubstitutionPosition));
306+
});
307+
it("classifies the termination of a multiline no substitution template", () => {
308+
testLexicalClassification("...`",
309+
ts.EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate,
310+
stringLiteral("...`"),
311+
finalEndOfLineState(ts.EndOfLineState.Start));
312+
});
313+
it("classifies the substitution parts and middle/tail of a multiline template string", () => {
314+
testLexicalClassification("${ 1 + 1 }...`",
315+
ts.EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate,
316+
stringLiteral("${"),
317+
numberLiteral("1"),
318+
operator("+"),
319+
numberLiteral("1"),
320+
stringLiteral("}...`"),
321+
finalEndOfLineState(ts.EndOfLineState.Start));
322+
});
323+
it("classifies a template middle and propagates the end of line state",() => {
324+
testLexicalClassification("${ 1 + 1 }...`",
325+
ts.EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate,
326+
stringLiteral("${"),
327+
numberLiteral("1"),
328+
operator("+"),
329+
numberLiteral("1"),
330+
stringLiteral("}...`"),
331+
finalEndOfLineState(ts.EndOfLineState.Start));
332+
});
333+
it("classifies substitution expressions with curly braces appropriately", () => {
334+
var pos = 0;
335+
var lastLength = 0;
336+
337+
testLexicalClassification("...${ () => { } } ${ { x: `1` } }...`",
338+
ts.EndOfLineState.InTemplateHeadOrNoSubstitutionTemplate,
339+
stringLiteral(track("...${"), pos),
340+
punctuation(track(" ", "("), pos),
341+
punctuation(track(")"), pos),
342+
punctuation(track(" ", "=>"), pos),
343+
punctuation(track(" ", "{"), pos),
344+
punctuation(track(" ", "}"), pos),
345+
stringLiteral(track(" ", "} ${"), pos),
346+
punctuation(track(" ", "{"), pos),
347+
identifier(track(" ", "x"), pos),
348+
punctuation(track(":"), pos),
349+
stringLiteral(track(" ", "`1`"), pos),
350+
punctuation(track(" ", "}"), pos),
351+
stringLiteral(track(" ", "}...`"), pos),
352+
finalEndOfLineState(ts.EndOfLineState.Start));
353+
354+
// Adjusts 'pos' by accounting for the length of each portion of the string,
355+
// but only return the last given string
356+
function track(...vals: string[]): string {
357+
for (var i = 0, n = vals.length; i < n; i++) {
358+
pos += lastLength;
359+
lastLength = vals[i].length;
360+
}
361+
return ts.lastOrUndefined(vals);
362+
}
363+
});
364+
257365
it("classifies partially written generics correctly.", function () {
258366
testLexicalClassification("Foo<number",
259367
ts.EndOfLineState.Start,

0 commit comments

Comments
 (0)