forked from microsoft/TypeScript
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathparser.ts
More file actions
4429 lines (3677 loc) · 220 KB
/
Copy pathparser.ts
File metadata and controls
4429 lines (3677 loc) · 220 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
///<reference path='references.ts' />
module TypeScript.Parser {
// The factory used to produce parse tree nodes. Injected normally by the
// TypeScript.Syntax.Abstract or TypeScript.Syntax.Conrete modules.
export var syntaxFactory: Syntax.ISyntaxFactory;
// Interface that represents the source that the parser pulls tokens from. Essentially, this
// is the interface that the parser needs an underlying scanner to provide. This allows us to
// separate out "what" the parser does with the tokens it retrieves versus "how" it obtains
// the tokens. i.e. all the logic for parsing language constructs sits in ParserImpl, while
// all the logic for retrieving tokens sits in individual IParserSources.
//
// By separating out this interface, we also make incremental parsing much easier. Instead of
// having the parser directly sit on top of the scanner, we sit it on this abstraction. Then
// in incremental scenarios, we can use the IncrementalParserSource to pull tokens (or even
// full nodes) from the previous tree when possible. Of course, we'll still end up using a
// scanner for new text. But that can all happen inside the source, with none of the logic in
// the parser having to be aware of it.
//
// In general terms, a parser source represents a position within a text. At that position,
// one can ask for the 'currentToken' that the source is pointing at. Then, once the parser
// consumes that token it can ask the source to 'moveToNextToken'.
//
// Additional special abilities include:
// 1) Being able to peek an arbitrary number of tokens ahead efficiently.
// 2) Being able to retrieve fully parsed nodes from the source, not just tokens. This happens
// in incremental scenarios when the source is certain that the node is completley safe to
// reuse.
// 3) Being able to get a 'rewind point' to the current location. This allows the parser to
// speculatively parse as much as it wants, and then reset itself back to that point,
// ensuring that no state changes that occurred after getting the 'rewing point' are
// observable.
// 4) Being able to reinterpret the current token being pointed at as a regular expression
// token. This is necessary as the scanner does not have enough information to correctly
// distinguish "/" or "/=" as divide tokens, versus "/..../" as a regex token. If the
// parser sees a "/" in a place where a divide is not allowed, but a regex would be, then
// it can call into the source and ask if a regex token could be returned instead. The
// sources are smart enough to do that and not be affected by any additional work they may
// have done when they originally scanned that token.
export interface IParserSource {
// The text we are parsing.
text: ISimpleText;
// the name of the file we're parsing.
fileName: string;
// The version of the language we're using while parsing. Does not affect the final tree,
// but can affect the diagnostics produced while parsing.
languageVersion: ts.ScriptTarget;
// The current syntax node the source is pointing at. Only available in incremental settings.
// The source can point at a node if that node doesn't intersect any of the text changes in
// the file, and doesn't contain certain unacceptable constructs. For example, if the node
// contains skipped text, then it will not be reused.
currentNode(): ISyntaxNode;
// The current token the source is pointing at.
currentToken(): ISyntaxToken;
// The current token reinterpretted contextually based on where the parser is. If the
// source is on a / or /= token, then it can be reinterpretted as a regex token. If the
// source is on a > token, it may be reinterpretted to: >> >>> >= >>= >>>=
currentContextualToken(): ISyntaxToken;
// Peek any number of tokens ahead from the current location in source. peekToken(0) is
// equivalent to 'currentToken', peekToken(1) is the next token, peekToken(2) the token
// after that, etc. If the caller peeks past the end of the text, then EndOfFile tokens
// will be returned.
peekToken(n: number): ISyntaxToken;
// Called to move the source to the next node or token once the parser has consumed the
// current one.
consumeNode(node: ISyntaxNode): void;
consumeToken(token: ISyntaxToken): void;
// Gets a rewind point that the parser can use to move back to after it speculatively
// parses something. The source guarantees that if the parser calls 'rewind' with that
// point that it will be mostly in the same state that it was in when 'getRewindPoint'
// was called. i.e. calling currentToken, peekToken, tokenDiagnostics, etc. will result
// in the same values. One allowed exemption to this is 'currentNode'. If a rewind point
// is requested and rewound, then getting the currentNode may not be possible. However,
// as this is purely a performance optimization, it will not affect correctness.
//
// Note: that rewind points are not free (but they should also not be too expensive). So
// they should be used judiciously. While a rewind point is held by the parser, the source
// is not free to do things that it would normally do. For example, it cannot throw away
// tokens that it has scanned on or after the rewind point as it must keep them alive for
// the parser to move back to.
//
// Rewind points also work in a stack fashion. The first rewind point given out must be
// the last rewind point released. Do not release them out of order, or bad things can
// happen.
//
// Do *NOT* forget to release a rewind point. Always put them in a finally block to ensure
// that they are released. If they are not released, things will still work, you will just
// consume far more memory than necessary.
getRewindPoint(): IRewindPoint;
// Rewinds the source to the position and state it was at when this rewind point was created.
// This does not need to be called if the parser decides it does not need to rewind. For
// example, the parser may speculatively parse out a lambda expression when it sees something
// ambiguous like "(a = b, c = ...". If it succeeds parsing that as a lambda, then it will
// just return that result. However, if it fails *then* it will rewind and try it again as
// a parenthesized expression.
rewind(rewindPoint: IRewindPoint): void;
// Called when the parser is done speculative parsing and no longer needs the rewind point.
// Must be called for every rewind point retrived.
releaseRewindPoint(rewindPoint: IRewindPoint): void;
// Retrieves the diagnostics generated while the source was producing nodes or tokens.
// Should generally only be called after the document has been completely parsed.
tokenDiagnostics(): Diagnostic[];
release(): void;
}
// Information the parser needs to effectively rewind.
export interface IRewindPoint {
}
var arrayPool: any[][] = [];
var arrayPoolCount: number = 0;
function getArray(): any[] {
if (arrayPoolCount === 0) {
return [];
}
arrayPoolCount--;
var result = arrayPool[arrayPoolCount];
arrayPool[arrayPoolCount] = null;
return result;
}
function returnZeroLengthArray(array: any[]) {
if (array.length === 0) {
returnArray(array);
}
}
function returnArray(array: any[]) {
array.length = 0;
arrayPool[arrayPoolCount] = array;
arrayPoolCount++;
}
interface IParserRewindPoint extends IRewindPoint {
// As we speculatively parse, we may build up diagnostics. When we rewind we want to
// 'forget' that information.In order to do that we store the count of diagnostics and
// when we start speculating, and we reset to that count when we're done. That way the
// speculative parse does not affect any further results.
diagnosticsCount: number;
// isInStrictMode and listParsingState should not have to be tracked by a rewind point.
// Because they are naturally mutated and restored based on the normal stack movement of
// the parser, they should automatically return to whatever value they had to begin with
// if the parser decides to rewind or not. However, to ensure that this is true, we track
// these variables and check if they have the same value when we're rewinding/releasing.
isInStrictMode: boolean;
listParsingState: ListParsingState;
}
// Contains the actual logic to parse typescript/javascript. This is the code that generally
// represents the logic necessary to handle all the language grammar constructs. When the
// language changes, this should generally only be the place necessary to fix up.
function createParseSyntaxTree(): (source: IParserSource, isDeclaration: boolean) => SyntaxTree {
// Name of the file we're parsing.
var fileName: string;
// Underlying source where we pull nodes and tokens from.
var source: IParserSource;
var languageVersion: ts.ScriptTarget;
// TODO: do we need to store/restore this when speculative parsing? I don't think so. The
// parsing logic already handles storing/restoring this and should work properly even if we're
// speculative parsing.
var listParsingState: number = 0;
// Whether or not we are in strict parsing mode. All that changes in strict parsing mode is
// that some tokens that would be considered identifiers may be considered keywords. When
// rewinding, we need to store and restore this as the mode may have changed.
//
// TODO: do we need to store/restore this when speculative parsing? I don't think so. The
// parsing logic already handles storing/restoring this and should work properly even if we're
// speculative parsing.
var isInStrictMode: boolean = false;
// Current state of the parser. If we need to rewind we will store and reset these values as
// appropriate.
// Diagnostics created when parsing invalid code. Any diagnosics created when speculative
// parsing need to removed when rewinding. To do this we store the count of diagnostics when
// we start speculative parsing. And if we rewind, we restore this to the same count that we
// started at.
var diagnostics: Diagnostic[] = [];
var parseNodeData: number = 0;
function parseSyntaxTree(_source: IParserSource, isDeclaration: boolean): SyntaxTree {
// First, set up our state.
fileName = _source.fileName;
source = _source;
languageVersion = source.languageVersion;
// Now actually parse the tree.
var result = parseSyntaxTreeWorker(isDeclaration);
// Now, clear out our state so that our singleton parser doesn't keep things alive.
diagnostics = [];
parseNodeData = SyntaxConstants.None;
fileName = null;
source.release();
source = null; _source = null;
return result;
}
function parseSyntaxTreeWorker(isDeclaration: boolean): SyntaxTree {
var sourceUnit = parseSourceUnit();
var allDiagnostics = source.tokenDiagnostics().concat(diagnostics);
allDiagnostics.sort((a: Diagnostic, b: Diagnostic) => a.start() - b.start());
return new SyntaxTree(syntaxFactory.isConcrete, sourceUnit, isDeclaration, allDiagnostics, fileName, source.text, languageVersion);
}
function getRewindPoint(): IParserRewindPoint {
var rewindPoint = <IParserRewindPoint>source.getRewindPoint();
rewindPoint.diagnosticsCount = diagnostics.length;
// Values we keep around for debug asserting purposes.
rewindPoint.isInStrictMode = isInStrictMode;
rewindPoint.listParsingState = listParsingState;
return rewindPoint;
}
function rewind(rewindPoint: IParserRewindPoint): void {
source.rewind(rewindPoint);
diagnostics.length = rewindPoint.diagnosticsCount;
}
function releaseRewindPoint(rewindPoint: IParserRewindPoint): void {
// Debug.assert(listParsingState === rewindPoint.listParsingState);
// Debug.assert(isInStrictMode === rewindPoint.isInStrictMode);
source.releaseRewindPoint(rewindPoint);
}
function currentNode(): ISyntaxNode {
var node = source.currentNode();
// We can only reuse a node if it was parsed under the same strict mode that we're
// currently in. i.e. if we originally parsed a node in non-strict mode, but then
// the user added 'using strict' at the top of the file, then we can't use that node
// again as the presense of strict mode may cause us to parse the tokens in the file
// differetly.
//
// Note: we *can* reuse tokens when the strict mode changes. That's because tokens
// are unaffected by strict mode. It's just the parser will decide what to do with it
// differently depending on what mode it is in.
if (node === null || parsedInStrictMode(node) !== isInStrictMode) {
return null;
}
return node;
}
function currentToken(): ISyntaxToken {
return source.currentToken();
}
function currentContextualToken(): ISyntaxToken {
// We're mutating the source here. We are potentially overwriting the original token we
// scanned with a regex token. So we have to clear our state.
return source.currentContextualToken();
}
function peekToken(n: number): ISyntaxToken {
return source.peekToken(n);
}
function consumeToken(token: ISyntaxToken): ISyntaxToken {
source.consumeToken(token);
return token;
}
function consumeNode(node: ISyntaxNode): void {
source.consumeNode(node);
}
//this method is called very frequently
//we should keep it simple so that it can be inlined.
function eatToken(kind: SyntaxKind): ISyntaxToken {
var token = currentToken();
if (token.kind() === kind) {
return consumeToken(token);
}
//slow part of EatToken(SyntaxKind kind)
return createMissingToken(kind, token);
}
// Eats the token if it is there. Otherwise does nothing. Will not report errors.
function tryEatToken(kind: SyntaxKind): ISyntaxToken {
var _currentToken = currentToken();
if (_currentToken.kind() === kind) {
return consumeToken(_currentToken);
}
return null;
}
// An identifier is basically any word, unless it is a reserved keyword. so 'foo' is an
// identifier and 'return' is not. Note: a word may or may not be an identifier depending
// on the state of the parser. For example, 'yield' is an identifier *unless* the parser
// is in strict mode.
function isIdentifier(token: ISyntaxToken): boolean {
var tokenKind = token.kind();
if (tokenKind === SyntaxKind.IdentifierName) {
return true;
}
// Keywords are only identifiers if they're FutureReservedStrictWords and we're in
// strict mode. *Or* if it's a typescript 'keyword'.
if (tokenKind >= SyntaxKind.FirstFutureReservedStrictKeyword) {
if (tokenKind <= SyntaxKind.LastFutureReservedStrictKeyword) {
// Could be a keyword or identifier. It's an identifier if we're not in strict
// mode.
return !isInStrictMode;
}
// If it's typescript keyword, then it's actually a javascript identifier.
return tokenKind <= SyntaxKind.LastTypeScriptKeyword;
}
// Anything else is not an identifier.
return false;
}
// This method should be called when the grammar calls for an *IdentifierName* and not an
// *Identifier*.
function eatIdentifierNameToken(): ISyntaxToken {
var token = currentToken();
// If we have an identifier name, then consume and return it.
var tokenKind = token.kind();
if (tokenKind === SyntaxKind.IdentifierName) {
return consumeToken(token);
}
// If we have a keyword, then it can be used as an identifier name. However, we need
// to convert it to an identifier so that no later parts of the systems see it as a
// keyword.
if (SyntaxFacts.isAnyKeyword(tokenKind)) {
return TypeScript.Syntax.convertKeywordToIdentifier(consumeToken(token));
}
return createMissingToken(SyntaxKind.IdentifierName, token);
}
function eatOptionalIdentifierToken(): ISyntaxToken {
return isIdentifier(currentToken()) ? eatIdentifierToken() : null;
}
// This method should be called when the grammar calls for an *Identifier* and not an
// *IdentifierName*.
function eatIdentifierToken(diagnosticCode?: string): ISyntaxToken {
var token = currentToken();
if (isIdentifier(token)) {
consumeToken(token);
if (token.kind() === SyntaxKind.IdentifierName) {
return token;
}
return TypeScript.Syntax.convertKeywordToIdentifier(token);
}
return createMissingToken(SyntaxKind.IdentifierName, token, diagnosticCode);
}
function previousTokenHasTrailingNewLine(token: ISyntaxToken): boolean {
var tokenFullStart = token.fullStart();
if (tokenFullStart === 0) {
// First token in the document. Thus it has no 'previous' token, and there is
// no preceding newline.
return false;
}
// If our previous token ended with a newline, then *by definition* we must have started
// at the beginning of a line.
var lineNumber = source.text.lineMap().getLineNumberFromPosition(tokenFullStart);
var lineStart = source.text.lineMap().getLineStartPosition(lineNumber);
return lineStart == tokenFullStart;
}
function canEatAutomaticSemicolon(allowWithoutNewLine: boolean): boolean {
var token = currentToken();
// An automatic semicolon is always allowed if we're at the end of the file.
var tokenKind = token.kind();
if (tokenKind === SyntaxKind.EndOfFileToken) {
return true;
}
// Or if the next token is a close brace (regardless of which line it is on).
if (tokenKind === SyntaxKind.CloseBraceToken) {
return true;
}
if (allowWithoutNewLine) {
return true;
}
// It is also allowed if there is a newline between the last token seen and the next one.
if (previousTokenHasTrailingNewLine(token)) {
return true;
}
return false;
}
function canEatExplicitOrAutomaticSemicolon(allowWithoutNewline: boolean): boolean {
var token = currentToken();
if (token.kind() === SyntaxKind.SemicolonToken) {
return true;
}
return canEatAutomaticSemicolon(allowWithoutNewline);
}
function eatExplicitOrAutomaticSemicolon(allowWithoutNewline: boolean): ISyntaxToken {
var token = currentToken();
// If we see a semicolon, then we can definitely eat it.
if (token.kind() === SyntaxKind.SemicolonToken) {
return consumeToken(token);
}
// Check if an automatic semicolon could go here. If so, then there's no problem and
// we can proceed without error. Return 'null' as there's no actual token for this
// position.
if (canEatAutomaticSemicolon(allowWithoutNewline)) {
return null;
}
// No semicolon could be consumed here at all. Just call the standard eating function
// so we get the token and the error for it.
return eatToken(SyntaxKind.SemicolonToken);
}
function createMissingToken(expectedKind: SyntaxKind, actual: ISyntaxToken, diagnosticCode?: string): ISyntaxToken {
var diagnostic = getExpectedTokenDiagnostic(expectedKind, actual, diagnosticCode);
addDiagnostic(diagnostic);
// The missing token will be at the full start of the current token. That way empty tokens
// will always be between real tokens and not inside an actual token.
return Syntax.emptyToken(expectedKind);
}
function getExpectedTokenDiagnostic(expectedKind: SyntaxKind, actual: ISyntaxToken, diagnosticCode: string): Diagnostic {
var token = currentToken();
var args: any[] = null;
// If a specialized diagnostic message was provided, just use that.
if (!diagnosticCode) {
// They wanted something specific, just report that that token was missing.
if (SyntaxFacts.isAnyKeyword(expectedKind) || SyntaxFacts.isAnyPunctuation(expectedKind)) {
diagnosticCode = DiagnosticCode._0_expected;
args = [SyntaxFacts.getText(expectedKind)];
}
else {
// They wanted an identifier.
// If the user supplied a keyword, give them a specialized message.
if (actual !== null && SyntaxFacts.isAnyKeyword(actual.kind())) {
diagnosticCode = DiagnosticCode.Identifier_expected_0_is_a_keyword;
args = [SyntaxFacts.getText(actual.kind())];
}
else {
// Otherwise just report that an identifier was expected.
diagnosticCode = DiagnosticCode.Identifier_expected;
}
}
}
return new Diagnostic(fileName, source.text.lineMap(), start(token, source.text), width(token), diagnosticCode, args);
}
function getBinaryExpressionPrecedence(tokenKind: SyntaxKind): BinaryExpressionPrecedence {
switch (tokenKind) {
case SyntaxKind.BarBarToken: return BinaryExpressionPrecedence.LogicalOrExpressionPrecedence;
case SyntaxKind.AmpersandAmpersandToken: return BinaryExpressionPrecedence.LogicalAndExpressionPrecedence;
case SyntaxKind.BarToken: return BinaryExpressionPrecedence.BitwiseOrExpressionPrecedence;
case SyntaxKind.CaretToken: return BinaryExpressionPrecedence.BitwiseExclusiveOrExpressionPrecedence;
case SyntaxKind.AmpersandToken: return BinaryExpressionPrecedence.BitwiseAndExpressionPrecedence;
case SyntaxKind.EqualsEqualsToken:
case SyntaxKind.ExclamationEqualsToken:
case SyntaxKind.EqualsEqualsEqualsToken:
case SyntaxKind.ExclamationEqualsEqualsToken:
return BinaryExpressionPrecedence.EqualityExpressionPrecedence;
case SyntaxKind.LessThanToken:
case SyntaxKind.GreaterThanToken:
case SyntaxKind.LessThanEqualsToken:
case SyntaxKind.GreaterThanEqualsToken:
case SyntaxKind.InstanceOfKeyword:
case SyntaxKind.InKeyword:
return BinaryExpressionPrecedence.RelationalExpressionPrecedence;
case SyntaxKind.LessThanLessThanToken:
case SyntaxKind.GreaterThanGreaterThanToken:
case SyntaxKind.GreaterThanGreaterThanGreaterThanToken:
return BinaryExpressionPrecedence.ShiftExpressionPrecdence;
case SyntaxKind.PlusToken:
case SyntaxKind.MinusToken:
return BinaryExpressionPrecedence.AdditiveExpressionPrecedence;
case SyntaxKind.AsteriskToken:
case SyntaxKind.SlashToken:
case SyntaxKind.PercentToken:
return BinaryExpressionPrecedence.MultiplicativeExpressionPrecedence;
}
throw Errors.invalidOperation();
}
function addSkippedTokenAfterNodeOrToken(nodeOrToken: ISyntaxNodeOrToken, skippedToken: ISyntaxToken): ISyntaxNodeOrToken {
if (isToken(nodeOrToken)) {
return addSkippedTokenAfterToken(<ISyntaxToken>nodeOrToken, skippedToken);
}
else if (isNode(nodeOrToken)) {
return addSkippedTokenAfterNode(<ISyntaxNode>nodeOrToken, skippedToken);
}
else {
throw Errors.invalidOperation();
}
}
function replaceTokenInParent(node: ISyntaxNode, oldToken: ISyntaxToken, newToken: ISyntaxToken): void {
// oldToken may be parented by a node or a list.
replaceTokenInParentWorker(oldToken, newToken);
var parent = oldToken.parent;
newToken.parent = parent;
// Walk upwards to our outermost node, clearing hte cached 'data' in it. This will
// make sure that the fullWidths and incrementally unusable bits are computed correctly
// when next requested.
while (true) {
// Parent must be a list or a node. All of those have a 'data' element.
Debug.assert(isNode(parent) || isList(parent) || isSeparatedList(parent));
var dataElement = <{ data: number }><any>parent;
if (dataElement.data) {
dataElement.data &= SyntaxConstants.NodeParsedInStrictModeMask
}
if (parent === node) {
break;
}
parent = parent.parent;
}
}
function replaceTokenInParentWorker(oldToken: ISyntaxToken, newToken: ISyntaxToken): void {
var parent = oldToken.parent;
if (isNode(parent)) {
var node = <any>parent;
for (var key in node) {
if (node[key] === oldToken) {
node[key] = newToken;
return;
}
}
}
else if (isList(parent)) {
var list1 = <ISyntaxNodeOrToken[]>parent;
for (var i = 0, n = list1.length; i < n; i++) {
if (list1[i] === oldToken) {
list1[i] = newToken;
return;
}
}
}
else if (isSeparatedList(parent)) {
var list2 = <ISyntaxNodeOrToken[]>parent;
for (var i = 0, n = childCount(list2); i < n; i++) {
if (childAt(list2, i) === oldToken) {
if (i % 2 === 0) {
list2[i / 2] = newToken;
}
else {
list2.separators[(i - 1) / 2] = newToken;
}
return;
}
}
}
throw Errors.invalidOperation();
}
function addSkippedTokenAfterNode(node: ISyntaxNode, skippedToken: ISyntaxToken): ISyntaxNode {
var oldToken = lastToken(node);
var newToken = addSkippedTokenAfterToken(oldToken, skippedToken);
replaceTokenInParent(node, oldToken, newToken);
return node;
}
function addSkippedTokensBeforeNode(node: ISyntaxNode, skippedTokens: ISyntaxToken[]): ISyntaxNode {
if (skippedTokens.length > 0) {
var oldToken = firstToken(node);
var newToken = addSkippedTokensBeforeToken(oldToken, skippedTokens);
replaceTokenInParent(node, oldToken, newToken);
}
return node;
}
function addSkippedTokensBeforeToken(token: ISyntaxToken, skippedTokens: ISyntaxToken[]): ISyntaxToken {
// Debug.assert(token.fullWidth() > 0 || token.kind() === SyntaxKind.EndOfFileToken);
// Debug.assert(skippedTokens.length > 0);
var leadingTrivia: ISyntaxTrivia[] = [];
for (var i = 0, n = skippedTokens.length; i < n; i++) {
var skippedToken = skippedTokens[i];
addSkippedTokenToTriviaArray(leadingTrivia, skippedToken);
}
addTriviaTo(token.leadingTrivia(source.text), leadingTrivia);
var updatedToken = Syntax.withLeadingTrivia(token, Syntax.triviaList(leadingTrivia), source.text);
// We've prepending this token with new leading trivia. This means the full start of
// the token is not where the scanner originally thought it was, but is instead at the
// start of the first skipped token.
updatedToken.setFullStart(skippedTokens[0].fullStart());
// Don't need this array anymore. Give it back so we can reuse it.
returnArray(skippedTokens);
return updatedToken;
}
function addSkippedTokensAfterToken(token: ISyntaxToken, skippedTokens: ISyntaxToken[]): ISyntaxToken {
// Debug.assert(token.fullWidth() > 0);
if (skippedTokens.length === 0) {
returnArray(skippedTokens);
return token;
}
var trailingTrivia = token.trailingTrivia(source.text).toArray();
for (var i = 0, n = skippedTokens.length; i < n; i++) {
addSkippedTokenToTriviaArray(trailingTrivia, skippedTokens[i]);
}
// Don't need this array anymore. Give it back so we can reuse it.
returnArray(skippedTokens);
return Syntax.withTrailingTrivia(token, Syntax.triviaList(trailingTrivia), source.text);
}
function addSkippedTokenAfterToken(token: ISyntaxToken, skippedToken: ISyntaxToken): ISyntaxToken {
// Debug.assert(token.fullWidth() > 0);
var trailingTrivia = token.trailingTrivia(source.text).toArray();
addSkippedTokenToTriviaArray(trailingTrivia, skippedToken);
return Syntax.withTrailingTrivia(token, Syntax.triviaList(trailingTrivia), source.text);
}
function addSkippedTokenToTriviaArray(array: ISyntaxTrivia[], skippedToken: ISyntaxToken): void {
// Debug.assert(skippedToken.text().length > 0);
// first, add the leading trivia of the skipped token to the array
addTriviaTo(skippedToken.leadingTrivia(source.text), array);
// now, add the text of the token as skipped text to the trivia array.
var trimmedToken = Syntax.withTrailingTrivia(Syntax.withLeadingTrivia(skippedToken, Syntax.emptyTriviaList, source.text), Syntax.emptyTriviaList, source.text);
// Because we removed the leading trivia from the skipped token, the full start of the
// trimmed token is the start of the skipped token.
trimmedToken.setFullStart(start(skippedToken, source.text));
array.push(Syntax.skippedTokenTrivia(trimmedToken, source.text));
// Finally, add the trailing trivia of the skipped token to the trivia array.
addTriviaTo(skippedToken.trailingTrivia(source.text), array);
}
function addTriviaTo(list: ISyntaxTriviaList, array: ISyntaxTrivia[]): void {
for (var i = 0, n = list.count(); i < n; i++) {
array.push(list.syntaxTriviaAt(i));
}
}
function setStrictMode(_isInStrictMode: boolean) {
isInStrictMode = _isInStrictMode;
parseNodeData = _isInStrictMode ? SyntaxConstants.NodeParsedInStrictModeMask : 0;
}
function parseSourceUnit(): SourceUnitSyntax {
var savedIsInStrictMode = isInStrictMode
var skippedTokens: ISyntaxToken[] = getArray();
var moduleElements = parseSyntaxList<IModuleElementSyntax>(ListParsingState.SourceUnit_ModuleElements, skippedTokens, updateStrictModeState);
setStrictMode(savedIsInStrictMode);
var sourceUnit = new syntaxFactory.SourceUnitSyntax(parseNodeData, moduleElements, currentToken());
sourceUnit = <SourceUnitSyntax>addSkippedTokensBeforeNode(sourceUnit, skippedTokens);
if (Debug.shouldAssert(AssertionLevel.Aggressive)) {
Debug.assert(fullWidth(sourceUnit) === source.text.length());
if (Debug.shouldAssert(AssertionLevel.VeryAggressive)) {
Debug.assert(fullText(sourceUnit) === source.text.substr(0, source.text.length()));
}
}
return sourceUnit;
}
function updateStrictModeState(items: any[]): void {
if (!isInStrictMode) {
// Check if all the items are directive prologue elements.
for (var i = 0; i < items.length; i++) {
var item = items[i];
if (!SyntaxFacts.isDirectivePrologueElement(item)) {
return;
}
}
setStrictMode(SyntaxFacts.isUseStrictDirective(items[items.length - 1]));
}
}
function isModuleElement(inErrorRecovery: boolean): boolean {
if (SyntaxUtilities.isModuleElement(currentNode())) {
return true;
}
var _modifierCount = modifierCount();
return isInterfaceEnumClassModuleImportOrExport(_modifierCount) ||
isStatement(_modifierCount, inErrorRecovery);
}
function tryParseModuleElement(inErrorRecovery: boolean): IModuleElementSyntax {
var node = currentNode();
if (SyntaxUtilities.isModuleElement(node)) {
consumeNode(node);
return <IModuleElementSyntax>node;
}
var _currentToken = currentToken();
var _modifierCount = modifierCount();
if (_modifierCount) {
// if we have modifiers, then these are definitely TS constructs and we can
// immediately start parsing them.
switch (peekToken(_modifierCount).kind()) {
case SyntaxKind.ImportKeyword: return parseImportDeclaration();
case SyntaxKind.ModuleKeyword: return parseModuleDeclaration();
case SyntaxKind.InterfaceKeyword: return parseInterfaceDeclaration();
case SyntaxKind.ClassKeyword: return parseClassDeclaration();
case SyntaxKind.EnumKeyword: return parseEnumDeclaration();
}
}
// No modifiers. If we see 'class, enum, import and export' we could technically
// aggressively consume them as they can't start another construct. However, it's
// not uncommon in error recovery to run into a situation where we see those keywords,
// but the code was using it as the name of an object property. To avoid overzealously
// consuming these, we only parse them out if we can see enough context to 'prove' that
// they really do start the module element
var nextToken = peekToken(1);
var currentTokenKind = _currentToken.kind();
switch (currentTokenKind) {
case SyntaxKind.ModuleKeyword:
if (isIdentifier(nextToken) || nextToken.kind() === SyntaxKind.StringLiteral) {
return parseModuleDeclaration();
}
break;
case SyntaxKind.ImportKeyword:
if (isIdentifier(nextToken)) {
return parseImportDeclaration();
}
break;
case SyntaxKind.ClassKeyword:
if (isIdentifier(nextToken)) {
return parseClassDeclaration();
}
break;
case SyntaxKind.EnumKeyword:
if (isIdentifier(nextToken)) {
return parseEnumDeclaration();
}
break;
case SyntaxKind.InterfaceKeyword:
if (isIdentifier(nextToken)) {
return parseInterfaceDeclaration();
}
break;
case SyntaxKind.ExportKeyword:
// 'export' could be a modifier on a statement (like export var ...). So we
// only want to parse out an export assignment here if we actually see the equals.
if (nextToken.kind() === SyntaxKind.EqualsToken) {
return parseExportAssignment();
}
break;
}
return tryParseStatementWorker(_currentToken, currentTokenKind, _modifierCount, inErrorRecovery);
}
function parseImportDeclaration(): ImportDeclarationSyntax {
return new syntaxFactory.ImportDeclarationSyntax(parseNodeData,
parseModifiers(), eatToken(SyntaxKind.ImportKeyword), eatIdentifierToken(), eatToken(SyntaxKind.EqualsToken), parseModuleReference(), eatExplicitOrAutomaticSemicolon(/*allowWithoutNewline:*/ false));
}
function parseExportAssignment(): ExportAssignmentSyntax {
return new syntaxFactory.ExportAssignmentSyntax(parseNodeData,
eatToken(SyntaxKind.ExportKeyword), eatToken(SyntaxKind.EqualsToken), eatIdentifierToken(), eatExplicitOrAutomaticSemicolon(/*allowWithoutNewline:*/ false));
}
function parseModuleReference(): IModuleReferenceSyntax {
return isExternalModuleReference() ? parseExternalModuleReference() : parseModuleNameModuleReference();
}
function isExternalModuleReference(): boolean {
return currentToken().kind() === SyntaxKind.RequireKeyword &&
peekToken(1).kind() === SyntaxKind.OpenParenToken;
}
function parseExternalModuleReference(): ExternalModuleReferenceSyntax {
return new syntaxFactory.ExternalModuleReferenceSyntax(parseNodeData,
eatToken(SyntaxKind.RequireKeyword), eatToken(SyntaxKind.OpenParenToken), eatToken(SyntaxKind.StringLiteral), eatToken(SyntaxKind.CloseParenToken));
}
function parseModuleNameModuleReference(): ModuleNameModuleReferenceSyntax {
return new syntaxFactory.ModuleNameModuleReferenceSyntax(parseNodeData, parseName(/*allowIdentifierNames:*/ false));
}
function tryParseTypeArgumentList(inExpression: boolean): TypeArgumentListSyntax {
var _currentToken = currentToken();
if (_currentToken.kind() !== SyntaxKind.LessThanToken) {
return null;
}
if (!inExpression) {
// if we're not in an expression, this must be a type argument list. Just parse
// it out as such.
var lessThanToken = consumeToken(_currentToken);
var skippedTokens: ISyntaxToken[] = getArray();
var typeArguments = parseSeparatedSyntaxList<ITypeSyntax>(ListParsingState.TypeArgumentList_Types, skippedTokens);
lessThanToken = addSkippedTokensAfterToken(lessThanToken, skippedTokens);
return new syntaxFactory.TypeArgumentListSyntax(parseNodeData, lessThanToken, typeArguments, eatToken(SyntaxKind.GreaterThanToken));
}
// If we're in an expression, then we only want to consume this as a type argument list
// if we're sure that it's a type arg list and not an arithmetic expression.
var rewindPoint = getRewindPoint();
// We've seen a '<'. Try to parse it out as a type argument list.
var lessThanToken = consumeToken(_currentToken);
var skippedTokens: ISyntaxToken[] = getArray();
var typeArguments = parseSeparatedSyntaxList<ITypeSyntax>(ListParsingState.TypeArgumentList_Types, skippedTokens);
var lessThanToken = addSkippedTokensAfterToken(lessThanToken, skippedTokens);
var greaterThanToken = eatToken(SyntaxKind.GreaterThanToken);
// We're in a context where '<' could be the start of a type argument list, or part
// of an arithmetic expression. We'll presume it's the latter unless we see the '>'
// and a following token that guarantees that it's supposed to be a type argument list.
if (greaterThanToken.fullWidth() === 0 || !canFollowTypeArgumentListInExpression(currentToken().kind())) {
rewind(rewindPoint);
releaseRewindPoint(rewindPoint);
return null;
}
else {
releaseRewindPoint(rewindPoint);
return new syntaxFactory.TypeArgumentListSyntax(parseNodeData, lessThanToken, typeArguments, greaterThanToken);
}
}
function canFollowTypeArgumentListInExpression(kind: SyntaxKind): boolean {
switch (kind) {
case SyntaxKind.OpenParenToken: // foo<x>(
case SyntaxKind.DotToken: // foo<x>.
// These two cases are the only cases where this token can legally follow a
// type argument list. So we definitely want to treat this as a type arg list.
case SyntaxKind.CloseParenToken: // foo<x>)
case SyntaxKind.CloseBracketToken: // foo<x>]
case SyntaxKind.ColonToken: // foo<x>:
case SyntaxKind.SemicolonToken: // foo<x>;
case SyntaxKind.CommaToken: // foo<x>,
case SyntaxKind.QuestionToken: // foo<x>?
case SyntaxKind.EqualsEqualsToken: // foo<x> ==
case SyntaxKind.EqualsEqualsEqualsToken: // foo<x> ===
case SyntaxKind.ExclamationEqualsToken: // foo<x> !=
case SyntaxKind.ExclamationEqualsEqualsToken: // foo<x> !==
case SyntaxKind.AmpersandAmpersandToken: // foo<x> &&
case SyntaxKind.BarBarToken: // foo<x> ||
case SyntaxKind.CaretToken: // foo<x> ^
case SyntaxKind.AmpersandToken: // foo<x> &
case SyntaxKind.BarToken: // foo<x> |
case SyntaxKind.CloseBraceToken: // foo<x> }
case SyntaxKind.EndOfFileToken: // foo<x>
// these cases can't legally follow a type arg list. However, they're not legal
// expressions either. The user is probably in the middle of a generic type. So
// treat it as such.
return true;
default:
// Anything else treat as an expression.
return false;
}
}
function parseName(allowIdentifierName: boolean): INameSyntax {
return tryParseName(allowIdentifierName) || eatIdentifierToken();
}
function eatRightSideOfName(allowIdentifierNames: boolean): ISyntaxToken {
var _currentToken = currentToken();
// Technically a keyword is valid here as all keywords are identifier names.
// However, often we'll encounter this in error situations when the keyword
// is actually starting another valid construct.
// So, we check for the following specific case:
// name.
// keyword identifierNameOrKeyword
// Note: the newlines are important here. For example, if that above code
// were rewritten into:
// name.keyword
// identifierNameOrKeyword
// Then we would consider it valid. That's because ASI would take effect and
// the code would be implicitly: "name.keyword; identifierNameOrKeyword".
// In the first case though, ASI will not take effect because there is not a
// line terminator after the keyword.
if (SyntaxFacts.isAnyKeyword(_currentToken.kind()) &&
previousTokenHasTrailingNewLine(_currentToken)) {
var token1 = peekToken(1);
if (!existsNewLineBetweenTokens(_currentToken, token1, source.text) &&
SyntaxFacts.isIdentifierNameOrAnyKeyword(token1)) {
return createMissingToken(SyntaxKind.IdentifierName, _currentToken);
}
}
return allowIdentifierNames ? eatIdentifierNameToken() : eatIdentifierToken();
}
function tryParseName(allowIdentifierNames: boolean): INameSyntax {
var token0 = currentToken();
var shouldContinue = isIdentifier(token0);
if (!shouldContinue) {
return null;
}
// Call eatIdentifierName to convert the token to an identifier if it is as keyword.
var current: INameSyntax = eatIdentifierToken();
while (shouldContinue && currentToken().kind() === SyntaxKind.DotToken) {
var dotToken = consumeToken(currentToken());
var identifierName = eatRightSideOfName(allowIdentifierNames);
current = new syntaxFactory.QualifiedNameSyntax(parseNodeData, current, dotToken, identifierName);
shouldContinue = identifierName.fullWidth() > 0;
}
return current;
}