take token kind from the tree in case if token kind from scanner is different

vladima · vladima · commit 7358b0f7d337 · 2014-12-01T13:29:49.000-08:00
diff --git a/src/services/formatting/formattingScanner.ts b/src/services/formatting/formattingScanner.ts
@@ -146,7 +146,7 @@ module ts.formatting {
             if (lastTokenInfo && expectedScanAction === lastScanAction) {
                 // readTokenInfo was called before with the same expected scan action.
                 // No need to re-scan text, return existing 'lastTokenInfo'
-                return lastTokenInfo;
+                return fixTokenKind(lastTokenInfo, n);
             }
 
             if (scanner.getStartPos() !== savedPos) {
@@ -207,17 +207,30 @@ module ts.formatting {
                 }
             }
 
-            return lastTokenInfo = {
+            lastTokenInfo = {
                 leadingTrivia: leadingTrivia,
                 trailingTrivia: trailingTrivia,
                 token: token
             }
+
+            return fixTokenKind(lastTokenInfo, n);
         }
 
         function isOnToken(): boolean {
             var current = (lastTokenInfo && lastTokenInfo.token.kind) ||  scanner.getToken();
             var startPos = (lastTokenInfo && lastTokenInfo.token.pos) || scanner.getStartPos();
             return startPos < endPos && current !== SyntaxKind.EndOfFileToken && !isTrivia(current);
         }
+
+        // when containing node in the tree is token 
+        // but its kind differs from the kind that was returned by the scanner,
+        // then kind needs to be fixed. This might happen in cases 
+        // when parser interprets token differently, i.e keyword treated as identifier
+        function fixTokenKind(tokenInfo: TokenInfo, container: Node): TokenInfo {
+            if (isToken(container) && tokenInfo.token.kind !== container.kind) {
+                tokenInfo.token.kind = container.kind;
+            }
+            return tokenInfo;
+        }
     }
 }
diff --git a/tests/cases/fourslash/formattingKeywordAsIdentifier.ts b/tests/cases/fourslash/formattingKeywordAsIdentifier.ts
@@ -0,0 +1,7 @@
+/// <reference path='fourslash.ts'/>
+
+////declare var module/*1*/
+
+goTo.marker("1");
+edit.insert(";");
+verify.currentLineContentIs("declare var module;");

Original file line number	Diff line number	Diff line change
`@@ -146,7 +146,7 @@ module ts.formatting {`
`146`	`146`	`if (lastTokenInfo && expectedScanAction === lastScanAction) {`
`147`	`147`	`// readTokenInfo was called before with the same expected scan action.`
`148`	`148`	`// No need to re-scan text, return existing 'lastTokenInfo'`
`149`		`- return lastTokenInfo;`
	`149`	`+ return fixTokenKind(lastTokenInfo, n);`
`150`	`150`	`}`
`151`	`151`
`152`	`152`	`if (scanner.getStartPos() !== savedPos) {`
`@@ -207,17 +207,30 @@ module ts.formatting {`
`207`	`207`	`}`
`208`	`208`	`}`
`209`	`209`
`210`		`- return lastTokenInfo = {`
	`210`	`+ lastTokenInfo = {`
`211`	`211`	`leadingTrivia: leadingTrivia,`
`212`	`212`	`trailingTrivia: trailingTrivia,`
`213`	`213`	`token: token`
`214`	`214`	`}`
	`215`	`+`
	`216`	`+ return fixTokenKind(lastTokenInfo, n);`
`215`	`217`	`}`
`216`	`218`
`217`	`219`	`function isOnToken(): boolean {`
`218`	`220`	`var current = (lastTokenInfo && lastTokenInfo.token.kind) \|\| scanner.getToken();`
`219`	`221`	`var startPos = (lastTokenInfo && lastTokenInfo.token.pos) \|\| scanner.getStartPos();`
`220`	`222`	`return startPos < endPos && current !== SyntaxKind.EndOfFileToken && !isTrivia(current);`
`221`	`223`	`}`
	`224`	`+`
	`225`	`+ // when containing node in the tree is token`
	`226`	`+ // but its kind differs from the kind that was returned by the scanner,`
	`227`	`+ // then kind needs to be fixed. This might happen in cases`
	`228`	`+ // when parser interprets token differently, i.e keyword treated as identifier`
	`229`	`+ function fixTokenKind(tokenInfo: TokenInfo, container: Node): TokenInfo {`
	`230`	`+ if (isToken(container) && tokenInfo.token.kind !== container.kind) {`
	`231`	`+ tokenInfo.token.kind = container.kind;`
	`232`	`+ }`
	`233`	`+ return tokenInfo;`
	`234`	`+ }`
`222`	`235`	`}`
`223`	`236`	`}`