Skip to content

Commit 0199940

Browse files
mhegazymhegazy
committed
wire classifier to use new compiler implementation
1 parent 132cd27 commit 0199940

2 files changed

Lines changed: 140 additions & 79 deletions

File tree

‎src/compiler/types.ts‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -217,7 +217,9 @@ module ts {
217217
FirstKeyword = BreakKeyword,
218218
LastKeyword = StringKeyword,
219219
FirstFutureReservedWord = ImplementsKeyword,
220-
LastFutureReservedWord = YieldKeyword
220+
LastFutureReservedWord = YieldKeyword,
221+
FirstPunctuation= OpenBraceToken,
222+
LastPunctuation = CaretEqualsToken
221223
}
222224

223225
export enum NodeFlags {

‎src/services/services.ts‎

Lines changed: 137 additions & 78 deletions
Original file line numberDiff line numberDiff line change
@@ -2207,33 +2207,16 @@ module ts {
22072207
}
22082208

22092209
/// Classifier
2210-
22112210
export function createClassifier(host: Logger): Classifier {
2212-
var scanner: TypeScript.Scanner.IScanner;
2213-
var lastDiagnosticKey: string = null;
2214-
var noRegexTable: boolean[];
2215-
var reportDiagnostic = (position: number, fullWidth: number, key: string, args: any[]) => {
2216-
lastDiagnosticKey = key;
2217-
};
2218-
2219-
if (!noRegexTable) {
2220-
noRegexTable = [];
2221-
noRegexTable[TypeScript.SyntaxKind.IdentifierName] = true;
2222-
noRegexTable[TypeScript.SyntaxKind.StringLiteral] = true;
2223-
noRegexTable[TypeScript.SyntaxKind.NumericLiteral] = true;
2224-
noRegexTable[TypeScript.SyntaxKind.RegularExpressionLiteral] = true;
2225-
noRegexTable[TypeScript.SyntaxKind.ThisKeyword] = true;
2226-
noRegexTable[TypeScript.SyntaxKind.PlusPlusToken] = true;
2227-
noRegexTable[TypeScript.SyntaxKind.MinusMinusToken] = true;
2228-
noRegexTable[TypeScript.SyntaxKind.CloseParenToken] = true;
2229-
noRegexTable[TypeScript.SyntaxKind.CloseBracketToken] = true;
2230-
noRegexTable[TypeScript.SyntaxKind.CloseBraceToken] = true;
2231-
noRegexTable[TypeScript.SyntaxKind.TrueKeyword] = true;
2232-
noRegexTable[TypeScript.SyntaxKind.FalseKeyword] = true;
2233-
}
2211+
var scanner: Scanner;
2212+
var noRegexTable: boolean[];
2213+
if (!noRegexTable) { noRegexTable = []; noRegexTable[SyntaxKind.Identifier] = true; noRegexTable[SyntaxKind.StringLiteral] = true; noRegexTable[SyntaxKind.NumericLiteral] = true; noRegexTable[SyntaxKind.RegularExpressionLiteral] = true; noRegexTable[SyntaxKind.ThisKeyword] = true; noRegexTable[SyntaxKind.PlusPlusToken] = true; noRegexTable[SyntaxKind.MinusMinusToken] = true; noRegexTable[SyntaxKind.CloseParenToken] = true; noRegexTable[SyntaxKind.CloseBracketToken] = true; noRegexTable[SyntaxKind.CloseBraceToken] = true; noRegexTable[SyntaxKind.TrueKeyword] = true; noRegexTable[SyntaxKind.FalseKeyword] = true; }
22342214

22352215
function getClassificationsForLine(text: string, lexState: EndOfLineState): ClassificationResult {
22362216
var offset = 0;
2217+
var lastTokenOrCommentEnd = 0;
2218+
var inMultiLineComment = false;
2219+
22372220
if (lexState !== EndOfLineState.Start) {
22382221
// If we're in a string literal, then prepend: "\
22392222
// (and a newline). That way when we lex we'll think we're still in a string literal.
@@ -2258,94 +2241,170 @@ module ts {
22582241
entries: []
22592242
};
22602243

2261-
var simpleText = TypeScript.SimpleText.fromString(text);
2262-
scanner = TypeScript.Scanner.createScanner(ScriptTarget.ES5, simpleText, reportDiagnostic);
2244+
scanner = createScanner(ScriptTarget.ES5, text, onError, processComment);
22632245

2264-
var lastTokenKind = TypeScript.SyntaxKind.None;
2265-
var token: TypeScript.ISyntaxToken = null;
2246+
var lastToken = SyntaxKind.Unknown;
2247+
var token = SyntaxKind.Unknown;
22662248
do {
2267-
lastDiagnosticKey = null;
2249+
inMultiLineComment = false;
2250+
2251+
token = scanner.scan();
2252+
2253+
if ((token === SyntaxKind.SlashToken || token === SyntaxKind.SlashEqualsToken) && !noRegexTable[lastToken]) {
2254+
if (scanner.reScanSlashToken() === SyntaxKind.RegularExpressionLiteral) {
2255+
token = SyntaxKind.RegularExpressionLiteral;
2256+
}
2257+
}
22682258

2269-
token = scanner.scan(!noRegexTable[lastTokenKind]);
2270-
lastTokenKind = token.kind();
2259+
lastToken = token;
22712260

2272-
processToken(text, simpleText, offset, token, result);
2261+
processToken();
22732262
}
2274-
while (token.kind() !== TypeScript.SyntaxKind.EndOfFileToken);
2263+
while (token !== SyntaxKind.EndOfFileToken);
22752264

2276-
lastDiagnosticKey = null;
22772265
return result;
2278-
}
22792266

2280-
function processToken(text: string, simpleText: TypeScript.ISimpleText, offset: number, token: TypeScript.ISyntaxToken, result: ClassificationResult): void {
2281-
processTriviaList(text, offset, token.leadingTrivia(simpleText), result);
2282-
addResult(text, offset, result, TypeScript.width(token), token.kind());
2283-
processTriviaList(text, offset, token.trailingTrivia(simpleText), result);
22842267

2285-
if (TypeScript.fullEnd(token) >= text.length) {
2286-
// We're at the end.
2287-
if (lastDiagnosticKey === TypeScript.DiagnosticCode.AsteriskSlash_expected) {
2288-
result.finalLexState = EndOfLineState.InMultiLineCommentTrivia;
2289-
return;
2290-
}
2268+
function onError(message: DiagnosticMessage): void {
2269+
inMultiLineComment = message.key === Diagnostics.Asterisk_Slash_expected.key;
2270+
}
2271+
2272+
function processComment(start: number, end: number) {
2273+
// add Leading white spaces
2274+
addLeadingWhiteSpace(start, end);
2275+
2276+
// add the comment
2277+
addResult(end - start, TokenClass.Comment);
2278+
}
2279+
2280+
function processToken(): void {
2281+
var start = scanner.getTokenPos();
2282+
var end = scanner.getTextPos();
2283+
2284+
// add Leading white spaces
2285+
addLeadingWhiteSpace(start, end);
2286+
2287+
// add the token
2288+
addResult(end - start, classFromKind(token));
22912289

2292-
if (token.kind() === TypeScript.SyntaxKind.StringLiteral) {
2293-
var tokenText = token.text();
2294-
if (tokenText.length > 0 && tokenText.charCodeAt(tokenText.length - 1) === TypeScript.CharacterCodes.backslash) {
2295-
var quoteChar = tokenText.charCodeAt(0);
2296-
result.finalLexState = quoteChar === TypeScript.CharacterCodes.doubleQuote
2297-
? EndOfLineState.InDoubleQuoteStringLiteral
2298-
: EndOfLineState.InSingleQuoteStringLiteral;
2290+
if (end >= text.length) {
2291+
// We're at the end.
2292+
if (inMultiLineComment) {
2293+
result.finalLexState = EndOfLineState.InMultiLineCommentTrivia;
22992294
return;
23002295
}
2296+
2297+
if (token === SyntaxKind.StringLiteral) {
2298+
var tokenText = scanner.getTokenText();
2299+
if (tokenText.length > 0 && tokenText.charCodeAt(tokenText.length - 1) === CharacterCodes.backslash) {
2300+
var quoteChar = tokenText.charCodeAt(0);
2301+
result.finalLexState = quoteChar === CharacterCodes.doubleQuote
2302+
? EndOfLineState.InDoubleQuoteStringLiteral
2303+
: EndOfLineState.InSingleQuoteStringLiteral;
2304+
return;
2305+
}
2306+
}
23012307
}
23022308
}
2303-
}
23042309

2305-
function processTriviaList(text: string, offset: number, triviaList: TypeScript.ISyntaxTriviaList, result: ClassificationResult): void {
2306-
for (var i = 0, n = triviaList.count(); i < n; i++) {
2307-
var trivia = triviaList.syntaxTriviaAt(i);
2308-
addResult(text, offset, result, trivia.fullWidth(), trivia.kind());
2310+
function addLeadingWhiteSpace(start: number, end: number): void {
2311+
if (start > lastTokenOrCommentEnd) {
2312+
addResult(start - lastTokenOrCommentEnd, TokenClass.Whitespace);
2313+
}
2314+
2315+
// Remeber the end of the last token
2316+
lastTokenOrCommentEnd = end;
23092317
}
2310-
}
23112318

2312-
function addResult(text: string, offset: number, result: ClassificationResult, length: number, kind: TypeScript.SyntaxKind): void {
2313-
if (length > 0) {
2314-
// If this is the first classification we're adding to the list, then remove any
2315-
// offset we have if we were continuing a construct from the previous line.
2316-
if (result.entries.length === 0) {
2317-
length -= offset;
2319+
function addResult(length: number, classification: TokenClass): void {
2320+
if (length > 0) {
2321+
// If this is the first classification we're adding to the list, then remove any
2322+
// offset we have if we were continuing a construct from the previous line.
2323+
if (result.entries.length === 0) {
2324+
length -= offset;
2325+
}
2326+
2327+
result.entries.push({ length: length, classification: classification });
23182328
}
2329+
}
2330+
}
23192331

2320-
result.entries.push({ length: length, classification: classFromKind(kind) });
2332+
function isBinaryExpressionOperatorToken(tokenKind: SyntaxKind): boolean {
2333+
switch (tokenKind) {
2334+
case SyntaxKind.AsteriskToken:
2335+
case SyntaxKind.SlashToken:
2336+
case SyntaxKind.PercentToken:
2337+
case SyntaxKind.PlusToken:
2338+
case SyntaxKind.MinusToken:
2339+
case SyntaxKind.LessThanLessThanToken:
2340+
case SyntaxKind.GreaterThanGreaterThanToken:
2341+
case SyntaxKind.GreaterThanGreaterThanGreaterThanToken:
2342+
case SyntaxKind.LessThanToken:
2343+
case SyntaxKind.GreaterThanToken:
2344+
case SyntaxKind.LessThanEqualsToken:
2345+
case SyntaxKind.GreaterThanEqualsToken:
2346+
case SyntaxKind.InstanceOfKeyword:
2347+
case SyntaxKind.InKeyword:
2348+
case SyntaxKind.EqualsEqualsToken:
2349+
case SyntaxKind.ExclamationEqualsToken:
2350+
case SyntaxKind.EqualsEqualsEqualsToken:
2351+
case SyntaxKind.ExclamationEqualsEqualsToken:
2352+
case SyntaxKind.AmpersandToken:
2353+
case SyntaxKind.CaretToken:
2354+
case SyntaxKind.BarToken:
2355+
case SyntaxKind.AmpersandAmpersandToken:
2356+
case SyntaxKind.BarBarToken:
2357+
case SyntaxKind.BarEqualsToken:
2358+
case SyntaxKind.AmpersandEqualsToken:
2359+
case SyntaxKind.CaretEqualsToken:
2360+
case SyntaxKind.LessThanLessThanEqualsToken:
2361+
case SyntaxKind.GreaterThanGreaterThanEqualsToken:
2362+
case SyntaxKind.GreaterThanGreaterThanGreaterThanEqualsToken:
2363+
case SyntaxKind.PlusEqualsToken:
2364+
case SyntaxKind.MinusEqualsToken:
2365+
case SyntaxKind.AsteriskEqualsToken:
2366+
case SyntaxKind.SlashEqualsToken:
2367+
case SyntaxKind.PercentEqualsToken:
2368+
case SyntaxKind.EqualsToken:
2369+
case SyntaxKind.CommaToken:
2370+
return true;
2371+
default: return false;
2372+
}
2373+
}
2374+
2375+
function isPrefixUnaryExpressionOperatorToken(tokenKind: SyntaxKind): boolean {
2376+
switch (tokenKind) {
2377+
case SyntaxKind.PlusToken:
2378+
case SyntaxKind.MinusToken:
2379+
case SyntaxKind.TildeToken:
2380+
case SyntaxKind.ExclamationToken:
2381+
case SyntaxKind.PlusPlusToken:
2382+
case SyntaxKind.MinusMinusToken:
2383+
return true;
2384+
default:
2385+
return false;
23212386
}
23222387
}
23232388

2324-
function classFromKind(kind: TypeScript.SyntaxKind) {
2325-
if (TypeScript.SyntaxFacts.isAnyKeyword(kind)) {
2389+
function classFromKind(kind: SyntaxKind) {
2390+
if (kind >= SyntaxKind.FirstKeyword && kind <= SyntaxKind.LastKeyword) {
23262391
return TokenClass.Keyword;
23272392
}
2328-
else if (TypeScript.SyntaxFacts.isBinaryExpressionOperatorToken(kind) ||
2329-
TypeScript.SyntaxFacts.isPrefixUnaryExpressionOperatorToken(kind)) {
2393+
else if (isBinaryExpressionOperatorToken(kind) || isPrefixUnaryExpressionOperatorToken(kind)) {
23302394
return TokenClass.Operator;
23312395
}
2332-
else if (TypeScript.SyntaxFacts.isAnyPunctuation(kind)) {
2396+
else if (kind >= SyntaxKind.FirstPunctuation && kind <= SyntaxKind.LastPunctuation) {
23332397
return TokenClass.Punctuation;
23342398
}
23352399

23362400
switch (kind) {
2337-
case TypeScript.SyntaxKind.WhitespaceTrivia:
2338-
return TokenClass.Whitespace;
2339-
case TypeScript.SyntaxKind.MultiLineCommentTrivia:
2340-
case TypeScript.SyntaxKind.SingleLineCommentTrivia:
2341-
return TokenClass.Comment;
2342-
case TypeScript.SyntaxKind.NumericLiteral:
2401+
case SyntaxKind.NumericLiteral:
23432402
return TokenClass.NumberLiteral;
2344-
case TypeScript.SyntaxKind.StringLiteral:
2403+
case SyntaxKind.StringLiteral:
23452404
return TokenClass.StringLiteral;
2346-
case TypeScript.SyntaxKind.RegularExpressionLiteral:
2405+
case SyntaxKind.RegularExpressionLiteral:
23472406
return TokenClass.RegExpLiteral;
2348-
case TypeScript.SyntaxKind.IdentifierName:
2407+
case SyntaxKind.Identifier:
23492408
default:
23502409
return TokenClass.Identifier;
23512410
}

0 commit comments

Comments
 (0)