LargeTokenFullWidthShift = 6,

LargeTokenLeadingTriviaShift = 3,

WhitespaceTrivia = 0x01, // 00000001

NewlineTrivia = 0x02, // 00000010

CommentTrivia = 0x04, // 00000100

TriviaMask = 0x07, // 00000111

KindMask = 0x7F, // 01111111

IsVariableWidthMask = 0x80, // 10000000

// These tokens are contextually created based on parsing decisions. We can't reuse

// them in incremental scenarios as we may be in a context where the parser would not

// create them.

switch (token.kind) {

// Created by the parser when it sees / or /= in a location where it needs an expression.

case SyntaxKind.RegularExpressionLiteral:

// Created by the parser when it sees > in a binary expression operator context.

case SyntaxKind.GreaterThanGreaterThanToken:

case SyntaxKind.GreaterThanGreaterThanGreaterThanToken:

case SyntaxKind.GreaterThanEqualsToken:

case SyntaxKind.GreaterThanGreaterThanEqualsToken:

case SyntaxKind.GreaterThanGreaterThanGreaterThanEqualsToken:

return true;

// Created by the parser when it sees } while parsing a template expression.

case SyntaxKind.TemplateMiddleToken:

case SyntaxKind.TemplateEndToken:

return true;

default:

return token.isKeywordConvertedToIdentifier();

}

if (lastTokenInfoTokenID !== syntaxID(token)) {

triviaScanner.fillTokenInfo(token, text, lastTokenInfo);

lastTokenInfoTokenID = syntaxID(token);

}

if (!token.hasLeadingTrivia()) {

return Syntax.emptyTriviaList;

}

return triviaScanner.scanTrivia(token, text, /*isTrailing:*/ false);

if (!token.hasTrailingTrivia()) {

return Syntax.emptyTriviaList;

}

return triviaScanner.scanTrivia(token, text, /*isTrailing:*/ true);

if (!token.hasLeadingTrivia()) {

return 0;

}

fillSizeInfo(token, text);

return lastTokenInfo.leadingTriviaWidth;

if (!token.hasTrailingTrivia()) {

return 0;

}

fillSizeInfo(token, text);

return token.fullWidth() - lastTokenInfo.leadingTriviaWidth - lastTokenInfo.width;

// No scanner tokens make their *containing node* incrementally unusable.

// Note: several scanner tokens may themselves be unusable. i.e. if the parser asks

// for a full node, then that ndoe can be returned even if it contains parser generated

// tokens (like regexs and merged operator tokens). However, if the parser asks for a

// for a token, then those contextual tokens will not be reusable.

return false;

public _primaryExpressionBrand: any; public _memberExpressionBrand: any; public _leftHandSideExpressionBrand: any; public _postfixExpressionBrand: any; public _unaryExpressionBrand: any; public _expressionBrand: any; public _typeBrand: any; public _nameBrand: any; public _propertyAssignmentBrand: any; public _propertyNameBrand: any;

public parent: ISyntaxElement;

public childCount: number;

constructor(private _fullStart: number, public kind: SyntaxKind) {

}

public setFullStart(fullStart: number): void {

this._fullStart = fullStart;

}

public childAt(index: number): ISyntaxElement { throw Errors.invalidOperation() }

public accept(visitor: ISyntaxVisitor): any { return visitor.visitToken(this) }

public isIncrementallyUnusable(): boolean { return false; }

public isKeywordConvertedToIdentifier(): boolean { return false; }

public hasSkippedToken(): boolean { return false; }

public fullText(): string { return SyntaxFacts.getText(this.kind); }

public text(): string { return this.fullText(); }

public leadingTrivia(): ISyntaxTriviaList { return Syntax.emptyTriviaList; }

public trailingTrivia(): ISyntaxTriviaList { return Syntax.emptyTriviaList; }

public leadingTriviaWidth(): number { return 0; }

public trailingTriviaWidth(): number { return 0; }

public fullWidth(): number { return fixedWidthTokenLength(this.kind); }

public fullStart(): number { return this._fullStart; }

public hasLeadingTrivia(): boolean { return false; }

public hasTrailingTrivia(): boolean { return false; }

public hasLeadingComment(): boolean { return false; }

public hasTrailingComment(): boolean { return false; }

public clone(): ISyntaxToken { return new FixedWidthTokenWithNoTrivia(this._fullStart, this.kind); }

public _primaryExpressionBrand: any; public _memberExpressionBrand: any; public _leftHandSideExpressionBrand: any; public _postfixExpressionBrand: any; public _unaryExpressionBrand: any; public _expressionBrand: any; public _typeBrand: any; public _nameBrand: any; public _propertyAssignmentBrand: any; public _propertyNameBrand: any;

public parent: ISyntaxElement;

public childCount: number;

private cachedText: string;

constructor(private _fullStart: number, public kind: SyntaxKind, private _packedFullWidthAndInfo: number, cachedText: string) {

if (cachedText !== undefined) {

this.cachedText = cachedText;

}

}

public setFullStart(fullStart: number): void {

this._fullStart = fullStart;

}

public childAt(index: number): ISyntaxElement { throw Errors.invalidOperation() }

public accept(visitor: ISyntaxVisitor): any { return visitor.visitToken(this) }

private syntaxTreeText(text: ISimpleText) {

var result = text || syntaxTree(this).text;

Debug.assert(result);

return result;

}

public isIncrementallyUnusable(): boolean { return tokenIsIncrementallyUnusable(this); }

public isKeywordConvertedToIdentifier(): boolean { return false; }

public hasSkippedToken(): boolean { return false; }

public fullText(text?: ISimpleText): string {

return fullText(this, this.syntaxTreeText(text));

}

public text(): string {

var cachedText = this.cachedText;

return cachedText !== undefined ? cachedText : SyntaxFacts.getText(this.kind);

}

public leadingTrivia(text?: ISimpleText): ISyntaxTriviaList { return leadingTrivia(this, this.syntaxTreeText(text)); }

public trailingTrivia(text?: ISimpleText): ISyntaxTriviaList { return trailingTrivia(this, this.syntaxTreeText(text)); }

public leadingTriviaWidth(text?: ISimpleText): number {

return leadingTriviaWidth(this, this.syntaxTreeText(text));

}

public trailingTriviaWidth(text?: ISimpleText): number {

return trailingTriviaWidth(this, this.syntaxTreeText(text));

}

public fullWidth(): number { return largeTokenUnpackFullWidth(this._packedFullWidthAndInfo); }

public fullStart(): number { return this._fullStart; }

public hasLeadingTrivia(): boolean { return largeTokenUnpackHasLeadingTrivia(this._packedFullWidthAndInfo); }

public hasTrailingTrivia(): boolean { return largeTokenUnpackHasTrailingTrivia(this._packedFullWidthAndInfo); }

public hasLeadingComment(): boolean { return largeTokenUnpackHasLeadingComment(this._packedFullWidthAndInfo); }

public hasTrailingComment(): boolean { return largeTokenUnpackHasTrailingComment(this._packedFullWidthAndInfo); }

public clone(): ISyntaxToken { return new LargeScannerToken(this._fullStart, this.kind, this._packedFullWidthAndInfo, this.cachedText); }

var scanner = createScannerInternal(languageVersion, text, reportDiagnostic);

return {

setIndex: scanner.setIndex,

scan: scanner.scan,

};

var str: string;

var index: number;

var start: number;

var end: number;

function setIndex(_index: number) {

index = _index;

}

function reset(_text: ISimpleText, _start: number, _end: number) {

var textLength = _text.length();

Debug.assert(_start <= textLength, "Token's start was not within the bounds of text.");

Debug.assert(_end <= textLength, "Token's end was not within the bounds of text:");

if (!str || text !== _text) {

text = _text;

str = _text.substr(0, textLength);

}

start = _start;

end = _end;

index = _start;

}

function scan(allowContextualToken: boolean): ISyntaxToken {

var fullStart = index;

var leadingTriviaInfo = scanTriviaInfo(/*isTrailing: */ false);

var start = index;

var kindAndIsVariableWidth = scanSyntaxKind(allowContextualToken);

var end = index;

var trailingTriviaInfo = scanTriviaInfo(/*isTrailing: */true);

var fullWidth = index - fullStart;

// If we have no trivia, and we are a fixed width token kind, and our size isn't too

// large, and we're a real fixed width token (and not something like "\u0076ar").

var kind = kindAndIsVariableWidth & ScannerConstants.KindMask;

var isFixedWidth = kind >= SyntaxKind.FirstFixedWidth && kind <= SyntaxKind.LastFixedWidth &&

((kindAndIsVariableWidth & ScannerConstants.IsVariableWidthMask) === 0);

if (isFixedWidth &&

leadingTriviaInfo === 0 && trailingTriviaInfo === 0) {

return new FixedWidthTokenWithNoTrivia(fullStart, kind);

}

else {

var packedFullWidthAndInfo = largeTokenPackData(fullWidth, leadingTriviaInfo, trailingTriviaInfo);

var cachedText = isFixedWidth ? undefined : text.substr(start, end - start);

return new LargeScannerToken(fullStart, kind, packedFullWidthAndInfo, cachedText);

}

}

function scanTrivia(parent: IScannerToken, text: ISimpleText, isTrailing: boolean): ISyntaxTriviaList {

var tokenFullStart = parent.fullStart();

var tokenStart = tokenFullStart + leadingTriviaWidth(parent, text)

if (isTrailing) {

reset(text, tokenStart + parent.text().length, tokenFullStart + parent.fullWidth());

}

else {

reset(text, tokenFullStart, tokenStart);

}

// Debug.assert(length > 0);

// Keep this exactly in sync with scanTriviaInfo

var trivia: ISyntaxTrivia[] = [];

while (true) {

if (index < end) {

var ch = str.charCodeAt(index);

switch (ch) {

// Unicode 3.0 space characters

case CharacterCodes.space:

case CharacterCodes.nonBreakingSpace:

case CharacterCodes.enQuad:

case CharacterCodes.emQuad:

case CharacterCodes.enSpace:

case CharacterCodes.emSpace:

case CharacterCodes.threePerEmSpace:

case CharacterCodes.fourPerEmSpace:

case CharacterCodes.sixPerEmSpace:

case CharacterCodes.figureSpace:

case CharacterCodes.punctuationSpace:

case CharacterCodes.thinSpace:

case CharacterCodes.hairSpace:

case CharacterCodes.zeroWidthSpace:

case CharacterCodes.narrowNoBreakSpace:

case CharacterCodes.ideographicSpace:

case CharacterCodes.tab:

case CharacterCodes.verticalTab:

case CharacterCodes.formFeed:

case CharacterCodes.byteOrderMark:

// Normal whitespace. Consume and continue.

trivia.push(scanWhitespaceTrivia());

continue;

case CharacterCodes.slash:

// Potential comment. Consume if so. Otherwise, break out and return.

var ch2 = str.charCodeAt(index + 1);

if (ch2 === CharacterCodes.slash) {

trivia.push(scanSingleLineCommentTrivia());

continue;

}

if (ch2 === CharacterCodes.asterisk) {

trivia.push(scanMultiLineCommentTrivia());

continue;

}

// Not a comment. Don't consume.

throw Errors.invalidOperation();

case CharacterCodes.carriageReturn:

case CharacterCodes.lineFeed:

case CharacterCodes.paragraphSeparator:

case CharacterCodes.lineSeparator:

trivia.push(scanLineTerminatorSequenceTrivia(ch));

// If we're consuming leading trivia, then we will continue consuming more

// trivia (including newlines) up to the first token we see. If we're

// consuming trailing trivia, then we break after the first newline we see.

if (!isTrailing) {

continue;

}

break;

default:

throw Errors.invalidOperation();

}

}

// Debug.assert(trivia.length > 0);

var triviaList = Syntax.triviaList(trivia);

triviaList.parent = parent;

return triviaList;

}

}

// Returns 0 if there was no trivia, or 1 if there was trivia. Returned as an int instead

// of a boolean because we'll need a numerical value later on to store in our tokens.

function scanTriviaInfo(isTrailing: boolean): number {

// Keep this exactly in sync with scanTrivia

var result = 0;

var _end = end;

while (index < _end) {

var ch = str.charCodeAt(index);

switch (ch) {

case CharacterCodes.tab:

case CharacterCodes.space:

case CharacterCodes.verticalTab:

case CharacterCodes.formFeed:

index++;

// we have trivia

result |= ScannerConstants.WhitespaceTrivia;

continue;

case CharacterCodes.carriageReturn:

if ((index + 1) < end && str.charCodeAt(index + 1) === CharacterCodes.lineFeed) {

index++;

}

// fall through.

case CharacterCodes.lineFeed:

case CharacterCodes.paragraphSeparator:

case CharacterCodes.lineSeparator:

index++;

// we have trivia

result |= ScannerConstants.NewlineTrivia;

// If we're consuming leading trivia, then we will continue consuming more

// trivia (including newlines) up to the first token we see. If we're

// consuming trailing trivia, then we break after the first newline we see.

if (isTrailing) {

return result;

}

continue;

case CharacterCodes.slash:

if ((index + 1) < _end) {

var ch2 = str.charCodeAt(index + 1);

if (ch2 === CharacterCodes.slash) {

// we have a comment, and we have trivia

result |= ScannerConstants.CommentTrivia;

skipSingleLineCommentTrivia();

continue;

}

if (ch2 === CharacterCodes.asterisk) {

// we have a comment, and we have trivia

result |= ScannerConstants.CommentTrivia;

skipMultiLineCommentTrivia();

continue;

}

}

// Not a comment. Don't consume.

return result;

default:

if (ch > CharacterCodes.maxAsciiCharacter && slowScanWhitespaceTriviaInfo(ch)) {

result |= ScannerConstants.WhitespaceTrivia;

continue;

}

return result;

}

}

return result;

}

function slowScanWhitespaceTriviaInfo(ch: number): boolean {

switch (ch) {

case CharacterCodes.nonBreakingSpace:

case CharacterCodes.enQuad:

case CharacterCodes.emQuad:

case CharacterCodes.enSpace:

case CharacterCodes.emSpace:

case CharacterCodes.threePerEmSpace:

case CharacterCodes.fourPerEmSpace:

case CharacterCodes.sixPerEmSpace:

case CharacterCodes.figureSpace:

case CharacterCodes.punctuationSpace:

case CharacterCodes.thinSpace:

case CharacterCodes.hairSpace:

case CharacterCodes.zeroWidthSpace:

case CharacterCodes.narrowNoBreakSpace:

case CharacterCodes.ideographicSpace:

case CharacterCodes.byteOrderMark:

index++;

return true;

default:

return false;

}

}

function isNewLineCharacter(ch: number): boolean {

switch (ch) {

case CharacterCodes.carriageReturn:

case CharacterCodes.lineFeed:

case CharacterCodes.paragraphSeparator:

case CharacterCodes.lineSeparator:

return true;

default:

return false;

}

}

function scanWhitespaceTrivia(): ISyntaxTrivia {

// We're going to be extracting text out of sliding window. Make sure it can't move past

// this point.

var absoluteStartIndex = index;

while (true) {

var ch = str.charCodeAt(index);

switch (ch) {

// Unicode 3.0 space characters

case CharacterCodes.space:

case CharacterCodes.nonBreakingSpace:

case CharacterCodes.enQuad:

case CharacterCodes.emQuad:

case CharacterCodes.enSpace:

case CharacterCodes.emSpace:

case CharacterCodes.threePerEmSpace:

case CharacterCodes.fourPerEmSpace:

case CharacterCodes.sixPerEmSpace:

case CharacterCodes.figureSpace:

case CharacterCodes.punctuationSpace:

case CharacterCodes.thinSpace:

case CharacterCodes.hairSpace:

case CharacterCodes.zeroWidthSpace:

case CharacterCodes.narrowNoBreakSpace:

case CharacterCodes.ideographicSpace:

case CharacterCodes.tab:

case CharacterCodes.verticalTab:

case CharacterCodes.formFeed:

case CharacterCodes.byteOrderMark:

// Normal whitespace. Consume and continue.

index++;

continue;

}

break;

}

return createTrivia(SyntaxKind.WhitespaceTrivia, absoluteStartIndex);

}

function createTrivia(kind: SyntaxKind, absoluteStartIndex: number): ISyntaxTrivia {

var fullWidth = index - absoluteStartIndex;

return Syntax.deferredTrivia(kind, text, absoluteStartIndex, fullWidth);

}

function scanSingleLineCommentTrivia(): ISyntaxTrivia {

var absoluteStartIndex = index;

skipSingleLineCommentTrivia();

return createTrivia(SyntaxKind.SingleLineCommentTrivia, absoluteStartIndex);

}

function skipSingleLineCommentTrivia(): void {

index += 2;

// The '2' is for the "//" we consumed.

while (index < end) {

if (isNewLineCharacter(str.charCodeAt(index))) {

return;

}

index++;

}

}

function scanMultiLineCommentTrivia(): ISyntaxTrivia {

var absoluteStartIndex = index;

skipMultiLineCommentTrivia();

return createTrivia(SyntaxKind.MultiLineCommentTrivia, absoluteStartIndex);

}

function skipMultiLineCommentTrivia(): number {

// The '2' is for the "/*" we consumed.

index += 2;

while (true) {

if (index === end) {

reportDiagnostic(end, 0, DiagnosticCode._0_expected, ["*/"]);

return;

}

if ((index + 1) < end &&

str.charCodeAt(index) === CharacterCodes.asterisk &&

str.charCodeAt(index + 1) === CharacterCodes.slash) {

index += 2;

return;

}

index++;

}

}

function scanLineTerminatorSequenceTrivia(ch: number): ISyntaxTrivia {

var absoluteStartIndex = index;

skipLineTerminatorSequence(ch);

return createTrivia(SyntaxKind.NewLineTrivia, absoluteStartIndex);

}

function skipLineTerminatorSequence(ch: number): void {

// Consume the first of the line terminator we saw.

index++;

// If it happened to be a \r and there's a following \n, then consume both.

if (ch === CharacterCodes.carriageReturn && str.charCodeAt(index) === CharacterCodes.lineFeed) {

index++;

}

}

function scanSyntaxKind(allowContextualToken: boolean): SyntaxKind {

if (index >= end) {

return SyntaxKind.EndOfFileToken;

}

var character = str.charCodeAt(index);

index++;

switch (character) {

case CharacterCodes.exclamation/*33*/: return scanExclamationToken();

case CharacterCodes.doubleQuote/*34*/: return scanStringLiteral(character);

case CharacterCodes.percent/*37*/: return scanPercentToken();

case CharacterCodes.ampersand/*38*/: return scanAmpersandToken();

case CharacterCodes.singleQuote/*39*/: return scanStringLiteral(character);

case CharacterCodes.openParen/*40*/: return SyntaxKind.OpenParenToken;

case CharacterCodes.closeParen/*41*/: return SyntaxKind.CloseParenToken;

case CharacterCodes.asterisk/*42*/: return scanAsteriskToken();

case CharacterCodes.plus/*43*/: return scanPlusToken();

case CharacterCodes.comma/*44*/: return SyntaxKind.CommaToken;

case CharacterCodes.minus/*45*/: return scanMinusToken();

case CharacterCodes.dot/*46*/: return scanDotToken();

case CharacterCodes.slash/*47*/: return scanSlashToken(allowContextualToken);

case CharacterCodes._0/*48*/: case CharacterCodes._1: case CharacterCodes._2: case CharacterCodes._3:

case CharacterCodes._4: case CharacterCodes._5: case CharacterCodes._6: case CharacterCodes._7:

case CharacterCodes._8: case CharacterCodes._9/*57*/:

return scanNumericLiteral(character);

case CharacterCodes.colon/*58*/: return SyntaxKind.ColonToken;

case CharacterCodes.semicolon/*59*/: return SyntaxKind.SemicolonToken;

case CharacterCodes.lessThan/*60*/: return scanLessThanToken();

case CharacterCodes.equals/*61*/: return scanEqualsToken();

case CharacterCodes.greaterThan/*62*/: return scanGreaterThanToken(allowContextualToken);

case CharacterCodes.question/*63*/: return SyntaxKind.QuestionToken;

case CharacterCodes.openBracket/*91*/: return SyntaxKind.OpenBracketToken;

case CharacterCodes.closeBracket/*93*/: return SyntaxKind.CloseBracketToken;

case CharacterCodes.caret/*94*/: return scanCaretToken();

case CharacterCodes.backtick/*96*/: return scanTemplateToken(character);

case CharacterCodes.openBrace/*123*/: return SyntaxKind.OpenBraceToken;

case CharacterCodes.bar/*124*/: return scanBarToken();

case CharacterCodes.closeBrace/*125*/: return scanCloseBraceToken(allowContextualToken, character);

case CharacterCodes.tilde/*126*/: return SyntaxKind.TildeToken;

}

// We run into so many identifiers (and keywords) when scanning, that we want the code to

// be as fast as possible. To that end, we have an extremely fast path for scanning that

// handles the 99.9% case of no-unicode characters and no unicode escapes.

if (isIdentifierStartCharacter[character]) {

var result = tryFastScanIdentifierOrKeyword(character);

if (result !== SyntaxKind.None) {

return result;

}

}

// Move the index back one and try the slow path.

index--;

if (isIdentifierStart(peekCharOrUnicodeEscape())) {

return slowScanIdentifierOrKeyword();

}

// Was nothing that we could understand. Report the issue and keep moving on.

var text = String.fromCharCode(character);

var messageText = getErrorMessageText(text);

reportDiagnostic(index, 1, DiagnosticCode.Unexpected_character_0, [messageText]);

index++;

return SyntaxKind.ErrorToken;

}

function isIdentifierStart(interpretedChar: number): boolean {

if (isIdentifierStartCharacter[interpretedChar]) {

return true;

}

return interpretedChar > CharacterCodes.maxAsciiCharacter && Unicode.isIdentifierStart(interpretedChar, languageVersion);

}

function isIdentifierPart(interpretedChar: number): boolean {

if (isIdentifierPartCharacter[interpretedChar]) {

return true;

}

return interpretedChar > CharacterCodes.maxAsciiCharacter && Unicode.isIdentifierPart(interpretedChar, languageVersion);

}

function tryFastScanIdentifierOrKeyword(firstCharacter: number): SyntaxKind {

var startIndex = index;

var character = firstCharacter;

// Note that we go up to the windowCount-1 so that we can read the character at the end

// of the window and check if it's *not* an identifier part character.

while (index < end) {

character = str.charCodeAt(index);

if (!isIdentifierPartCharacter[character]) {

break;

}

index++;

}

if (index < end && (character === CharacterCodes.backslash || character > CharacterCodes.maxAsciiCharacter)) {

// We saw a \ (which could start a unicode escape), or we saw a unicode character.

// This can't be scanned quickly. Don't update the window position and just bail out

// to the slow path.

index = startIndex;

return SyntaxKind.None;

}

else {

// Saw an ascii character that wasn't a backslash and wasn't an identifier

// character. Or we hit the end of the file This identifier is done.

// Also check if it a keyword if it started with a keyword start char.

if (isKeywordStartCharacter[firstCharacter]) {

return ScannerUtilities.identifierKind(str, startIndex - 1, index - startIndex + 1);

}

else {

return SyntaxKind.IdentifierName;

}

}

}

// A slow path for scanning identifiers. Called when we run into a unicode character or

// escape sequence while processing the fast path.

function slowScanIdentifierOrKeyword(): SyntaxKind {

var startIndex = index;

do {

scanCharOrUnicodeEscape();

}

while (isIdentifierPart(peekCharOrUnicodeEscape()));

// From ES6 specification.

// The ReservedWord definitions are specified as literal sequences of Unicode

// characters.However, any Unicode character in a ReservedWord can also be

// expressed by a \ UnicodeEscapeSequence that expresses that same Unicode

// character's code point.Use of such escape sequences does not change the meaning

// of the ReservedWord.

//

// i.e. "\u0076ar" is the keyword 'var'. Check for that here.

var length = index - startIndex;

var text = str.substr(startIndex, length);

var valueText = massageEscapes(text);

var keywordKind = SyntaxFacts.getTokenKind(valueText);

if (keywordKind >= SyntaxKind.FirstKeyword && keywordKind <= SyntaxKind.LastKeyword) {

// We have a keyword, but it is also variable width. We can't put represent this

// width a fixed width token.

return keywordKind | ScannerConstants.IsVariableWidthMask;

}

return SyntaxKind.IdentifierName;

}