Skip to content

Commit cc5e2fb

Browse files
alexdimaalexdima
committed
Handle most of the forceTokenization callers (microsoft#32508)
1 parent 2a34051 commit cc5e2fb

12 files changed

Lines changed: 69 additions & 37 deletions

File tree

‎src/vs/editor/common/commands/shiftCommand.ts‎

Lines changed: 20 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -112,28 +112,30 @@ export class ShiftCommand implements ICommand {
112112
if (contentStartVisibleColumn % tabSize !== 0) {
113113
// The current line is "miss-aligned", so let's see if this is expected...
114114
// This can only happen when it has trailing commas in the indent
115-
let enterAction = LanguageConfigurationRegistry.getRawEnterActionAtPosition(model, lineNumber - 1, model.getLineMaxColumn(lineNumber - 1));
116-
if (enterAction) {
117-
extraSpaces = previousLineExtraSpaces;
118-
if (enterAction.appendText) {
119-
for (let j = 0, lenJ = enterAction.appendText.length; j < lenJ && extraSpaces < tabSize; j++) {
120-
if (enterAction.appendText.charCodeAt(j) === CharCode.Space) {
121-
extraSpaces++;
122-
} else {
123-
break;
115+
if (model.isCheapToTokenize(lineNumber - 1)) {
116+
let enterAction = LanguageConfigurationRegistry.getRawEnterActionAtPosition(model, lineNumber - 1, model.getLineMaxColumn(lineNumber - 1));
117+
if (enterAction) {
118+
extraSpaces = previousLineExtraSpaces;
119+
if (enterAction.appendText) {
120+
for (let j = 0, lenJ = enterAction.appendText.length; j < lenJ && extraSpaces < tabSize; j++) {
121+
if (enterAction.appendText.charCodeAt(j) === CharCode.Space) {
122+
extraSpaces++;
123+
} else {
124+
break;
125+
}
124126
}
125127
}
126-
}
127-
if (enterAction.removeText) {
128-
extraSpaces = Math.max(0, extraSpaces - enterAction.removeText);
129-
}
128+
if (enterAction.removeText) {
129+
extraSpaces = Math.max(0, extraSpaces - enterAction.removeText);
130+
}
130131

131-
// Act as if `prefixSpaces` is not part of the indentation
132-
for (let j = 0; j < extraSpaces; j++) {
133-
if (indentationEndIndex === 0 || lineText.charCodeAt(indentationEndIndex - 1) !== CharCode.Space) {
134-
break;
132+
// Act as if `prefixSpaces` is not part of the indentation
133+
for (let j = 0; j < extraSpaces; j++) {
134+
if (indentationEndIndex === 0 || lineText.charCodeAt(indentationEndIndex - 1) !== CharCode.Space) {
135+
break;
136+
}
137+
indentationEndIndex--;
135138
}
136-
indentationEndIndex--;
137139
}
138140
}
139141
}

‎src/vs/editor/common/controller/cursorTypeOperations.ts‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -223,7 +223,7 @@ export class TypeOperations {
223223

224224
let lineText = model.getLineContent(selection.startLineNumber);
225225

226-
if (/^\s*$/.test(lineText)) {
226+
if (/^\s*$/.test(lineText) && model.isCheapToTokenize(selection.startLineNumber)) {
227227
let goodIndent = this._goodIndentForLine(config, model, selection.startLineNumber);
228228
goodIndent = goodIndent || '\t';
229229
let possibleTypeText = config.normalizeIndentation(goodIndent);
@@ -286,7 +286,7 @@ export class TypeOperations {
286286
}
287287

288288
private static _enter(config: CursorConfiguration, model: ITokenizedModel, keepPosition: boolean, range: Range): ICommand {
289-
if (model.getFirstInvalidLineNumber() < range.getStartPosition().lineNumber) {
289+
if (!model.isCheapToTokenize(range.getStartPosition().lineNumber)) {
290290
let lineText = model.getLineContent(range.startLineNumber);
291291
let indentation = strings.getLeadingWhitespace(lineText).substring(0, range.startColumn - 1);
292292
return TypeOperations._typeCommand(range, '\n' + config.normalizeIndentation(indentation), keepPosition);
@@ -380,9 +380,8 @@ export class TypeOperations {
380380
return false;
381381
}
382382

383-
let firstInvalidLineNumber = model.getFirstInvalidLineNumber();
384383
for (let i = 0, len = selections.length; i < len; i++) {
385-
if (firstInvalidLineNumber < selections[i].getEndPosition().lineNumber) {
384+
if (!model.isCheapToTokenize(selections[i].getEndPosition().lineNumber)) {
386385
return false;
387386
}
388387
}
@@ -528,6 +527,11 @@ export class TypeOperations {
528527
}
529528
}
530529

530+
if (!model.isCheapToTokenize(position.lineNumber)) {
531+
// Do not force tokenization
532+
return false;
533+
}
534+
531535
model.forceTokenization(position.lineNumber);
532536
const lineTokens = model.getLineTokens(position.lineNumber);
533537

@@ -607,8 +611,7 @@ export class TypeOperations {
607611
}
608612

609613
private static _isTypeInterceptorElectricChar(config: CursorConfiguration, model: ITokenizedModel, selections: Selection[]) {
610-
let firstInvalidLineNumber = model.getFirstInvalidLineNumber();
611-
if (selections.length === 1 && firstInvalidLineNumber >= selections[0].getEndPosition().lineNumber) {
614+
if (selections.length === 1 && model.isCheapToTokenize(selections[0].getEndPosition().lineNumber)) {
612615
return true;
613616
}
614617
return false;

‎src/vs/editor/common/editorCommon.ts‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -820,10 +820,19 @@ export interface ITokenizedModel extends ITextModel {
820820
forceTokenization(lineNumber: number): void;
821821

822822
/**
823-
* Get the line number of the first line whose tokens might be inaccurate.
823+
* If it is cheap, force tokenization information for `lineNumber` to be accurate.
824+
* This is based on a heuristic.
824825
* @internal
825826
*/
826-
getFirstInvalidLineNumber(): number;
827+
tokenizeIfCheap(lineNumber: number): void;
828+
829+
/**
830+
* Check if calling `forceTokenization` for this `lineNumber` will be cheap (time-wise).
831+
* This is based on a heuristic.
832+
* @internal
833+
*/
834+
isCheapToTokenize(lineNumber: number): boolean;
835+
827836
/**
828837
* Get the tokens for the line `lineNumber`.
829838
* The tokens might be inaccurate. Use `forceTokenization` to ensure accurate tokens.

‎src/vs/editor/common/model/textModelWithTokens.ts‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -169,10 +169,6 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
169169
return result;
170170
}
171171

172-
public getFirstInvalidLineNumber(): number {
173-
return this._invalidLineStartIndex + 1;
174-
}
175-
176172
public forceTokenization(lineNumber: number): void {
177173
if (lineNumber < 1 || lineNumber > this.getLineCount()) {
178174
throw new Error('Illegal value ' + lineNumber + ' for `lineNumber`');
@@ -183,6 +179,17 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
183179
});
184180
}
185181

182+
public isCheapToTokenize(lineNumber: number): boolean {
183+
const firstInvalidLineNumber = this._invalidLineStartIndex + 1;
184+
return (firstInvalidLineNumber >= lineNumber);
185+
}
186+
187+
public tokenizeIfCheap(lineNumber: number): void {
188+
if (this.isCheapToTokenize(lineNumber)) {
189+
this.forceTokenization(lineNumber);
190+
}
191+
}
192+
186193
public getLineTokens(lineNumber: number): LineTokens {
187194
if (lineNumber < 1 || lineNumber > this.getLineCount()) {
188195
throw new Error('Illegal value ' + lineNumber + ' for `lineNumber`');

‎src/vs/editor/contrib/comment/common/blockCommentCommand.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -135,7 +135,7 @@ export class BlockCommentCommand implements editorCommon.ICommand {
135135
var endLineNumber = this._selection.endLineNumber;
136136
var endColumn = this._selection.endColumn;
137137

138-
model.forceTokenization(startLineNumber);
138+
model.tokenizeIfCheap(startLineNumber);
139139
let languageId = model.getLanguageIdAtPosition(startLineNumber, startColumn);
140140
let config = LanguageConfigurationRegistry.getComments(languageId);
141141
if (!config || !config.blockCommentStartToken || !config.blockCommentEndToken) {

‎src/vs/editor/contrib/comment/common/lineCommentCommand.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ export class LineCommentCommand implements editorCommon.ICommand {
6464
*/
6565
public static _gatherPreflightCommentStrings(model: editorCommon.ITokenizedModel, startLineNumber: number, endLineNumber: number): ILinePreflightData[] {
6666

67-
model.forceTokenization(startLineNumber);
67+
model.tokenizeIfCheap(startLineNumber);
6868
const languageId = model.getLanguageIdAtPosition(startLineNumber, 1);
6969

7070
const config = LanguageConfigurationRegistry.getComments(languageId);
@@ -266,7 +266,7 @@ export class LineCommentCommand implements editorCommon.ICommand {
266266
* Given an unsuccessful analysis, delegate to the block comment command
267267
*/
268268
private _executeBlockComment(model: editorCommon.ITokenizedModel, builder: editorCommon.IEditOperationBuilder, s: Selection): void {
269-
model.forceTokenization(s.startLineNumber);
269+
model.tokenizeIfCheap(s.startLineNumber);
270270
let languageId = model.getLanguageIdAtPosition(s.startLineNumber, s.startColumn);
271271
let config = LanguageConfigurationRegistry.getComments(languageId);
272272
if (!config || !config.blockCommentStartToken || !config.blockCommentEndToken) {

‎src/vs/editor/contrib/indentation/common/indentation.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -423,7 +423,7 @@ export class AutoIndentOnPaste implements IEditorContribution {
423423
}
424424

425425
const model = this.editor.getModel();
426-
if (model.getFirstInvalidLineNumber() < range.getStartPosition().lineNumber) {
426+
if (!model.isCheapToTokenize(range.getStartPosition().lineNumber)) {
427427
return;
428428
}
429429
const { tabSize, insertSpaces } = model.getOptions();

‎src/vs/editor/contrib/linesOperations/common/moveLinesCommand.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -261,6 +261,7 @@ export class MoveLinesCommand implements ICommand {
261261
}
262262

263263
let maxColumn = model.getLineMaxColumn(validPrecedingLine);
264+
// TODO@Peng TODO@forceTokenization: getEnterAction forces tokenization
264265
let enter = LanguageConfigurationRegistry.getEnterAction(model, new Range(validPrecedingLine, maxColumn, validPrecedingLine, maxColumn));
265266

266267
if (enter) {

‎src/vs/editor/contrib/suggest/browser/suggestModel.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -293,7 +293,7 @@ export class SuggestModel implements IDisposable {
293293
} else if (quickSuggestions === true) {
294294
// all good
295295
} else {
296-
model.forceTokenization(pos.lineNumber);
296+
model.tokenizeIfCheap(pos.lineNumber);
297297
const { tokenType } = model
298298
.getLineTokens(pos.lineNumber)
299299
.findTokenAtOffset(pos.column - 1);

‎src/vs/editor/test/common/controller/cursor.test.ts‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3471,6 +3471,7 @@ suite('autoClosingPairs', () => {
34713471
const autoCloseColumns = extractSpecialColumns(model.getLineMaxColumn(lineNumber), autoClosePositions[i]);
34723472

34733473
for (let column = 1; column < autoCloseColumns.length; column++) {
3474+
model.forceTokenization(lineNumber);
34743475
if (autoCloseColumns[column] === ColumnType.Special1) {
34753476
assertType(model, cursor, lineNumber, column, '(', '()', `auto closes @ (${lineNumber}, ${column})`);
34763477
} else {
@@ -3513,6 +3514,7 @@ suite('autoClosingPairs', () => {
35133514
const autoCloseColumns = extractSpecialColumns(model.getLineMaxColumn(lineNumber), autoClosePositions[i]);
35143515

35153516
for (let column = 1; column < autoCloseColumns.length; column++) {
3517+
model.forceTokenization(lineNumber);
35163518
if (autoCloseColumns[column] === ColumnType.Special1) {
35173519
assertType(model, cursor, lineNumber, column, '\'', '\'\'', `auto closes @ (${lineNumber}, ${column})`);
35183520
} else if (autoCloseColumns[column] === ColumnType.Special2) {
@@ -3555,42 +3557,50 @@ suite('autoClosingPairs', () => {
35553557
}
35563558

35573559
// First gif
3560+
model.forceTokenization(model.getLineCount());
35583561
typeCharacters(cursor, 'teste1 = teste\' ok');
35593562
assert.equal(model.getLineContent(1), 'teste1 = teste\' ok');
35603563

35613564
cursor.setSelections('test', [new Selection(1, 1000, 1, 1000)]);
35623565
typeCharacters(cursor, '\n');
3566+
model.forceTokenization(model.getLineCount());
35633567
typeCharacters(cursor, 'teste2 = teste \'ok');
35643568
assert.equal(model.getLineContent(2), 'teste2 = teste \'ok\'');
35653569

35663570
cursor.setSelections('test', [new Selection(2, 1000, 2, 1000)]);
35673571
typeCharacters(cursor, '\n');
3572+
model.forceTokenization(model.getLineCount());
35683573
typeCharacters(cursor, 'teste3 = teste" ok');
35693574
assert.equal(model.getLineContent(3), 'teste3 = teste" ok');
35703575

35713576
cursor.setSelections('test', [new Selection(3, 1000, 3, 1000)]);
35723577
typeCharacters(cursor, '\n');
3578+
model.forceTokenization(model.getLineCount());
35733579
typeCharacters(cursor, 'teste4 = teste "ok');
35743580
assert.equal(model.getLineContent(4), 'teste4 = teste "ok"');
35753581

35763582
// Second gif
35773583
cursor.setSelections('test', [new Selection(4, 1000, 4, 1000)]);
35783584
typeCharacters(cursor, '\n');
3585+
model.forceTokenization(model.getLineCount());
35793586
typeCharacters(cursor, 'teste \'');
35803587
assert.equal(model.getLineContent(5), 'teste \'\'');
35813588

35823589
cursor.setSelections('test', [new Selection(5, 1000, 5, 1000)]);
35833590
typeCharacters(cursor, '\n');
3591+
model.forceTokenization(model.getLineCount());
35843592
typeCharacters(cursor, 'teste "');
35853593
assert.equal(model.getLineContent(6), 'teste ""');
35863594

35873595
cursor.setSelections('test', [new Selection(6, 1000, 6, 1000)]);
35883596
typeCharacters(cursor, '\n');
3597+
model.forceTokenization(model.getLineCount());
35893598
typeCharacters(cursor, 'teste\'');
35903599
assert.equal(model.getLineContent(7), 'teste\'');
35913600

35923601
cursor.setSelections('test', [new Selection(7, 1000, 7, 1000)]);
35933602
typeCharacters(cursor, '\n');
3603+
model.forceTokenization(model.getLineCount());
35943604
typeCharacters(cursor, 'teste"');
35953605
assert.equal(model.getLineContent(8), 'teste"');
35963606
});

0 commit comments

Comments
 (0)