Skip to content

Commit fa2fdbb

Browse files
alexdimaalexdima
committed
Simplify tokenIterator
1 parent 3ad82b9 commit fa2fdbb

5 files changed

Lines changed: 131 additions & 96 deletions

File tree

‎src/vs/editor/common/core/lineTokens.ts‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -132,16 +132,25 @@ export class LineTokens {
132132
}
133133

134134
public findTokenAtOffset(offset:number): LineToken {
135+
if (this._textLength === 0) {
136+
return null;
137+
}
135138
let tokenIndex = this.findTokenIndexAtOffset(offset);
136139
let modeIndex = ModeTransition.findIndexInSegmentsArray(this.modeTransitions, offset);
137140
return new LineToken(this, tokenIndex, modeIndex);
138141
}
139142

140-
public first(): LineToken {
143+
public firstToken(): LineToken {
144+
if (this._textLength === 0) {
145+
return null;
146+
}
141147
return new LineToken(this, 0, 0);
142148
}
143149

144-
public last(): LineToken {
150+
public lastToken(): LineToken {
151+
if (this._textLength === 0) {
152+
return null;
153+
}
145154
return new LineToken(this, this._tokens.length - 1, this.modeTransitions.length - 1);
146155
}
147156

‎src/vs/editor/common/editorCommon.ts‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@ import {Position} from 'vs/editor/common/core/position';
1818
import {Range} from 'vs/editor/common/core/range';
1919
import {Selection} from 'vs/editor/common/core/selection';
2020
import {ModeTransition} from 'vs/editor/common/core/modeTransition';
21-
import {Token} from 'vs/editor/common/core/token';
2221
import {IndentRange} from 'vs/editor/common/model/indentRanges';
2322
import {ICommandHandlerDescription} from 'vs/platform/commands/common/commands';
2423
import {ContextKeyExpr, RawContextKey} from 'vs/platform/contextkey/common/contextkey';
@@ -1246,7 +1245,7 @@ export interface IWordRange {
12461245
* @internal
12471246
*/
12481247
export interface ITokenInfo {
1249-
token: Token;
1248+
type: string;
12501249
lineNumber: number;
12511250
startColumn: number;
12521251
endColumn: number;

‎src/vs/editor/common/model/textModelWithTokens.ts‎

Lines changed: 15 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -527,6 +527,9 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
527527
const lineText = this._lines[lineNumber - 1].text;
528528

529529
const currentToken = lineTokens.findTokenAtOffset(position.column - 1);
530+
if (!currentToken) {
531+
return null;
532+
}
530533
const currentModeBrackets = LanguageConfigurationRegistry.getBracketsSupport(currentToken.modeId);
531534

532535
// If position is in between two tokens, try first looking in the previous token
@@ -624,11 +627,13 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
624627
currentToken = lineTokens.findTokenAtOffset(position.column - 1);
625628
searchStopOffset = position.column - 1;
626629
} else {
627-
currentToken = lineTokens.last();
628-
searchStopOffset = currentToken.endOffset;
630+
currentToken = lineTokens.lastToken();
631+
if (currentToken) {
632+
searchStopOffset = currentToken.endOffset;
633+
}
629634
}
630635

631-
do {
636+
while(currentToken) {
632637
if (currentToken.modeId === modeId && !ignoreBracketsInToken(currentToken.type)) {
633638

634639
while (true) {
@@ -658,8 +663,7 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
658663
if (currentToken) {
659664
searchStopOffset = currentToken.endOffset;
660665
}
661-
662-
} while(currentToken);
666+
}
663667
}
664668

665669
return null;
@@ -682,11 +686,13 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
682686
currentToken = lineTokens.findTokenAtOffset(position.column - 1);
683687
searchStartOffset = position.column - 1;
684688
} else {
685-
currentToken = lineTokens.first();
686-
searchStartOffset = currentToken.startOffset;
689+
currentToken = lineTokens.firstToken();
690+
if (currentToken) {
691+
searchStartOffset = currentToken.startOffset;
692+
}
687693
}
688694

689-
do {
695+
while (currentToken) {
690696
if (currentToken.modeId === modeId && !ignoreBracketsInToken(currentToken.type)) {
691697
while (true) {
692698
let r = BracketsUtils.findNextBracketInToken(bracketRegex, lineNumber, lineText, searchStartOffset, currentToken.endOffset);
@@ -715,7 +721,7 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
715721
if (currentToken) {
716722
searchStartOffset = currentToken.startOffset;
717723
}
718-
} while (currentToken);
724+
}
719725
}
720726

721727
return null;

‎src/vs/editor/common/model/tokenIterator.ts‎

Lines changed: 103 additions & 82 deletions
Original file line numberDiff line numberDiff line change
@@ -5,117 +5,138 @@
55
'use strict';
66

77
import * as editorCommon from 'vs/editor/common/editorCommon';
8-
import {Token} from 'vs/editor/common/core/token';
9-
import {LineTokens} from 'vs/editor/common/core/lineTokens';
10-
11-
export class TokenIterator implements editorCommon.ITokenIterator {
8+
import {LineToken} from 'vs/editor/common/core/lineTokens';
9+
import {Position} from 'vs/editor/common/core/position';
10+
11+
class TokenInfo implements editorCommon.ITokenInfo {
12+
_tokenInfoBrand: void;
13+
14+
_actual: LineToken;
15+
public lineNumber: number;
16+
public startColumn: number;
17+
public endColumn: number;
18+
public type: string;
19+
20+
constructor(actual:LineToken, lineNumber:number) {
21+
this._actual = actual;
22+
this.lineNumber = lineNumber;
23+
this.startColumn = this._actual.startOffset + 1;
24+
this.endColumn = this._actual.endOffset + 1;
25+
this.type = this._actual.type;
26+
}
27+
}
1228

13-
private _model:editorCommon.ITokenizedModel;
14-
private _currentLineNumber:number;
15-
private _currentTokenIndex:number;
16-
private _currentLineTokens:LineTokens;
17-
private _next:editorCommon.ITokenInfo;
18-
private _prev:editorCommon.ITokenInfo;
29+
function findClosestNonEmptyLine(model:editorCommon.ITokenizedModel, position:Position): Position {
30+
const lineNumber = position.lineNumber;
31+
if (model.getLineMaxColumn(lineNumber) !== 1) {
32+
return position;
33+
}
1934

20-
constructor(model:editorCommon.ITokenizedModel, position:editorCommon.IPosition) {
21-
this._model = model;
22-
this._currentLineNumber = position.lineNumber;
23-
this._currentTokenIndex = 0;
24-
this._readLineTokens(this._currentLineNumber);
25-
this._next = null;
26-
this._prev = null;
35+
const lineCount = model.getLineCount();
2736

28-
// start with a position to next/prev run
29-
var columnIndex = position.column - 1, tokenEndIndex = Number.MAX_VALUE;
37+
// we need to go up or down
38+
let distance = 1;
39+
while (true) {
40+
let aboveLineNumber = lineNumber - distance;
41+
let belowLineNumber = lineNumber + distance;
3042

31-
for (var i = this._currentLineTokens.getTokenCount() - 1; i >= 0; i--) {
32-
let tokenStartIndex = this._currentLineTokens.getTokenStartOffset(i);
43+
if (aboveLineNumber < 1 && belowLineNumber > lineCount) {
44+
// No more lines above or below
45+
break;
46+
}
3347

34-
if (tokenStartIndex <= columnIndex && columnIndex <= tokenEndIndex) {
48+
if (aboveLineNumber >= 1) {
49+
let aboveMaxColumn = model.getLineMaxColumn(aboveLineNumber);
50+
if (aboveMaxColumn !== 1) {
51+
// bingo!
52+
return new Position(aboveLineNumber, aboveMaxColumn);
53+
}
54+
}
3555

36-
this._currentTokenIndex = i;
37-
this._next = this._current();
38-
this._prev = this._current();
39-
break;
56+
if (belowLineNumber <= lineCount) {
57+
let belowMaxColumn = model.getLineMaxColumn(belowLineNumber);
58+
if (belowMaxColumn !== 1) {
59+
// bingo!
60+
return new Position(belowLineNumber, 1);
4061
}
41-
tokenEndIndex = tokenStartIndex;
4262
}
43-
}
4463

45-
private _readLineTokens(lineNumber:number): void {
46-
this._currentLineTokens = this._model.getLineTokens(lineNumber, false);
64+
distance++;
4765
}
66+
return null;
67+
}
4868

49-
private _advanceNext() {
50-
this._prev = this._next;
69+
export class TokenIterator implements editorCommon.ITokenIterator {
70+
71+
private _model:editorCommon.ITokenizedModel;
72+
private _lineCount:number;
73+
private _prev:TokenInfo;
74+
private _next:TokenInfo;
75+
76+
constructor(model:editorCommon.ITokenizedModel, position:Position) {
77+
this._model = model;
78+
this._lineCount = this._model.getLineCount();
79+
this._prev = null;
5180
this._next = null;
52-
if (this._currentTokenIndex + 1 < this._currentLineTokens.getTokenCount()) {
53-
// There are still tokens on current line
54-
this._currentTokenIndex++;
55-
this._next = this._current();
5681

57-
} else {
58-
// find the next line with tokens
59-
while (this._currentLineNumber + 1 <= this._model.getLineCount()) {
60-
this._currentLineNumber++;
61-
this._readLineTokens(this._currentLineNumber);
62-
if (this._currentLineTokens.getTokenCount() > 0) {
63-
this._currentTokenIndex = 0;
64-
this._next = this._current();
65-
break;
66-
}
67-
}
68-
if (this._next === null) {
69-
// prepare of a previous run
70-
this._readLineTokens(this._currentLineNumber);
71-
this._currentTokenIndex = this._currentLineTokens.getTokenCount();
72-
this._advancePrev();
73-
this._next = null;
82+
position = findClosestNonEmptyLine(model, position);
83+
if (position) {
84+
let lineTokens = this._model.getLineTokens(position.lineNumber);
85+
let currentToken = lineTokens.findTokenAtOffset(position.column - 1);
86+
if (currentToken) {
87+
this._prev = this._next = new TokenInfo(currentToken, position.lineNumber);
7488
}
7589
}
7690
}
7791

78-
private _advancePrev() {
79-
this._next = this._prev;
80-
this._prev = null;
81-
if (this._currentTokenIndex > 0) {
82-
// There are still tokens on current line
83-
this._currentTokenIndex--;
84-
this._prev = this._current();
92+
private _advanceNext(): void {
93+
if (!this._next) {
94+
return;
95+
}
96+
97+
let lineNumber = this._next.lineNumber;
98+
let next = this._next._actual.next();
99+
while (!next && lineNumber < this._lineCount) {
100+
lineNumber++;
101+
let currentLineTokens = this._model.getLineTokens(lineNumber);
102+
next = currentLineTokens.firstToken();
103+
}
85104

105+
this._prev = this._next;
106+
if (next) {
107+
this._next = new TokenInfo(next, lineNumber);
86108
} else {
87-
// find previous line with tokens
88-
while (this._currentLineNumber > 1) {
89-
this._currentLineNumber--;
90-
this._readLineTokens(this._currentLineNumber);
91-
if (this._currentLineTokens.getTokenCount() > 0) {
92-
this._currentTokenIndex = this._currentLineTokens.getTokenCount() - 1;
93-
this._prev = this._current();
94-
break;
95-
}
96-
}
109+
this._next = null;
97110
}
98111
}
99112

100-
private _current(): editorCommon.ITokenInfo {
101-
let startIndex = this._currentLineTokens.getTokenStartOffset(this._currentTokenIndex);
102-
let type = this._currentLineTokens.getTokenType(this._currentTokenIndex);
103-
let endIndex = this._currentLineTokens.getTokenEndOffset(this._currentTokenIndex);
113+
private _advancePrev(): void {
114+
if (!this._prev) {
115+
return;
116+
}
104117

105-
return {
106-
token: new Token(startIndex, type),
107-
lineNumber: this._currentLineNumber,
108-
startColumn: startIndex + 1,
109-
endColumn: endIndex + 1
110-
};
118+
let lineNumber = this._prev.lineNumber;
119+
let prev = this._prev._actual.prev();
120+
while (!prev && lineNumber > 1) {
121+
lineNumber--;
122+
let currentLineTokens = this._model.getLineTokens(lineNumber);
123+
prev = currentLineTokens.lastToken();
124+
}
125+
126+
this._next = this._prev;
127+
if (prev) {
128+
this._prev = new TokenInfo(prev, lineNumber);
129+
} else {
130+
this._prev = null;
131+
}
111132
}
112133

113134
public hasNext(): boolean {
114135
return this._next !== null;
115136
}
116137

117138
public next(): editorCommon.ITokenInfo {
118-
var result = this._next;
139+
const result = this._next;
119140
this._advanceNext();
120141
return result;
121142
}
@@ -125,7 +146,7 @@ export class TokenIterator implements editorCommon.ITokenIterator {
125146
}
126147

127148
public prev(): editorCommon.ITokenInfo {
128-
var result = this._prev;
149+
const result = this._prev;
129150
this._advancePrev();
130151
return result;
131152
}

‎src/vs/workbench/parts/themes/test/electron-browser/themes.test.contribution.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -125,7 +125,7 @@ class Snapper {
125125
let content = model.getValueInRange({ startLineNumber: lineNumber, endLineNumber: lineNumber, startColumn: tokenInfo.startColumn, endColumn: tokenInfo.endColumn});
126126
result.push({
127127
c: content,
128-
t: this.normalizeType(tokenInfo.token.type),
128+
t: this.normalizeType(tokenInfo.type),
129129
r: {}
130130
});
131131
}

0 commit comments

Comments
 (0)