Skip to content

Commit 6af3539

Browse files
alexdimaalexdima
committed
Keep previous partial tokens
1 parent 15aa2e1 commit 6af3539

5 files changed

Lines changed: 264 additions & 27 deletions

File tree

‎src/vs/editor/common/model.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -832,7 +832,7 @@ export interface ITextModel {
832832
/**
833833
* @internal
834834
*/
835-
setPartialSemanticTokens(tokens: MultilineTokens2[] | null): void;
835+
setPartialSemanticTokens(range: Range, tokens: MultilineTokens2[] | null): void;
836836

837837
/**
838838
* @internal

‎src/vs/editor/common/model/textModel.ts‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1807,11 +1807,17 @@ export class TextModel extends Disposable implements model.ITextModel {
18071807
return this._tokens2.isComplete();
18081808
}
18091809

1810-
public setPartialSemanticTokens(tokens: MultilineTokens2[]): void {
1810+
public setPartialSemanticTokens(range: Range, tokens: MultilineTokens2[]): void {
18111811
if (this.hasSemanticTokens()) {
18121812
return;
18131813
}
1814-
this.setSemanticTokens(tokens, false);
1814+
const changedRange = this._tokens2.setPartial(range, tokens);
1815+
1816+
this._emitModelTokensChangedEvent({
1817+
tokenizationSupportChanged: false,
1818+
semanticTokensApplied: true,
1819+
ranges: [{ fromLineNumber: changedRange.startLineNumber, toLineNumber: changedRange.endLineNumber }]
1820+
});
18151821
}
18161822

18171823
public tokenizeViewport(startLineNumber: number, endLineNumber: number): void {

‎src/vs/editor/common/model/tokensStore.ts‎

Lines changed: 127 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
import * as arrays from 'vs/base/common/arrays';
77
import { LineTokens } from 'vs/editor/common/core/lineTokens';
88
import { Position } from 'vs/editor/common/core/position';
9-
import { IRange } from 'vs/editor/common/core/range';
9+
import { IRange, Range } from 'vs/editor/common/core/range';
1010
import { ColorId, FontStyle, LanguageId, MetadataConsts, StandardTokenType, TokenMetadata } from 'vs/editor/common/modes';
1111
import { writeUInt32BE, readUInt32BE } from 'vs/base/common/buffer';
1212
import { CharCode } from 'vs/base/common/charCode';
@@ -124,16 +124,7 @@ export class MultilineTokensBuilder {
124124
}
125125
}
126126

127-
export interface IEncodedTokens {
128-
getMaxDeltaLine(): number;
129-
getLineTokens(deltaLine: number): LineTokens2 | null;
130-
131-
clear(): void;
132-
acceptDeleteRange(horizontalShiftForFirstLineTokens: number, startDeltaLine: number, startCharacter: number, endDeltaLine: number, endCharacter: number): void;
133-
acceptInsertText(deltaLine: number, character: number, eolCount: number, firstLineLength: number, lastLineLength: number, firstCharCode: number): void;
134-
}
135-
136-
export class SparseEncodedTokens implements IEncodedTokens {
127+
export class SparseEncodedTokens {
137128
/**
138129
* The encoding of tokens is:
139130
* 4*i deltaLine (from `startLineNumber`)
@@ -157,6 +148,17 @@ export class SparseEncodedTokens implements IEncodedTokens {
157148
return this._getDeltaLine(tokenCount - 1);
158149
}
159150

151+
public getRange(): Range | null {
152+
const tokenCount = this._getTokenCount();
153+
if (tokenCount === 0) {
154+
return null;
155+
}
156+
const startChar = this._getStartCharacter(0);
157+
const maxDeltaLine = this._getDeltaLine(tokenCount - 1);
158+
const endChar = this._getEndCharacter(tokenCount - 1);
159+
return new Range(0, startChar + 1, maxDeltaLine, endChar + 1);
160+
}
161+
160162
private _getTokenCount(): number {
161163
return this._tokenCount;
162164
}
@@ -165,6 +167,18 @@ export class SparseEncodedTokens implements IEncodedTokens {
165167
return this._tokens[4 * tokenIndex];
166168
}
167169

170+
private _getStartCharacter(tokenIndex: number): number {
171+
return this._tokens[4 * tokenIndex + 1];
172+
}
173+
174+
private _getEndCharacter(tokenIndex: number): number {
175+
return this._tokens[4 * tokenIndex + 2];
176+
}
177+
178+
public isEmpty(): boolean {
179+
return (this._getTokenCount() === 0);
180+
}
181+
168182
public getLineTokens(deltaLine: number): LineTokens2 | null {
169183
let low = 0;
170184
let high = this._getTokenCount() - 1;
@@ -201,6 +215,45 @@ export class SparseEncodedTokens implements IEncodedTokens {
201215
this._tokenCount = 0;
202216
}
203217

218+
public removeTokens(startDeltaLine: number, startChar: number, endDeltaLine: number, endChar: number): number {
219+
const tokens = this._tokens;
220+
const tokenCount = this._tokenCount;
221+
let newTokenCount = 0;
222+
let hasDeletedTokens = false;
223+
let firstDeltaLine = 0;
224+
for (let i = 0; i < tokenCount; i++) {
225+
const srcOffset = 4 * i;
226+
const tokenDeltaLine = tokens[srcOffset];
227+
const tokenStartCharacter = tokens[srcOffset + 1];
228+
const tokenEndCharacter = tokens[srcOffset + 2];
229+
const tokenMetadata = tokens[srcOffset + 3];
230+
231+
if (
232+
(tokenDeltaLine > startDeltaLine || (tokenDeltaLine === startDeltaLine && tokenStartCharacter >= startChar))
233+
&& (tokenDeltaLine < endDeltaLine || (tokenDeltaLine === endDeltaLine && tokenEndCharacter <= endChar))
234+
) {
235+
hasDeletedTokens = true;
236+
} else {
237+
if (newTokenCount === 0) {
238+
firstDeltaLine = tokenDeltaLine;
239+
}
240+
if (hasDeletedTokens) {
241+
// must move the token to the left
242+
const destOffset = 4 * newTokenCount;
243+
tokens[destOffset] = tokenDeltaLine - firstDeltaLine;
244+
tokens[destOffset + 1] = tokenStartCharacter;
245+
tokens[destOffset + 2] = tokenEndCharacter;
246+
tokens[destOffset + 3] = tokenMetadata;
247+
}
248+
newTokenCount++;
249+
}
250+
}
251+
252+
this._tokenCount = newTokenCount;
253+
254+
return firstDeltaLine;
255+
}
256+
204257
public acceptDeleteRange(horizontalShiftForFirstLineTokens: number, startDeltaLine: number, startCharacter: number, endDeltaLine: number, endCharacter: number): void {
205258
// This is a bit complex, here are the cases I used to think about this:
206259
//
@@ -457,9 +510,9 @@ export class MultilineTokens2 {
457510

458511
public startLineNumber: number;
459512
public endLineNumber: number;
460-
public tokens: IEncodedTokens;
513+
public tokens: SparseEncodedTokens;
461514

462-
constructor(startLineNumber: number, tokens: IEncodedTokens) {
515+
constructor(startLineNumber: number, tokens: SparseEncodedTokens) {
463516
this.startLineNumber = startLineNumber;
464517
this.tokens = tokens;
465518
this.endLineNumber = this.startLineNumber + this.tokens.getMaxDeltaLine();
@@ -469,13 +522,33 @@ export class MultilineTokens2 {
469522
this.endLineNumber = this.startLineNumber + this.tokens.getMaxDeltaLine();
470523
}
471524

525+
public isEmpty(): boolean {
526+
return this.tokens.isEmpty();
527+
}
528+
472529
public getLineTokens(lineNumber: number): LineTokens2 | null {
473530
if (this.startLineNumber <= lineNumber && lineNumber <= this.endLineNumber) {
474531
return this.tokens.getLineTokens(lineNumber - this.startLineNumber);
475532
}
476533
return null;
477534
}
478535

536+
public getRange(): Range | null {
537+
const deltaRange = this.tokens.getRange();
538+
if (!deltaRange) {
539+
return deltaRange;
540+
}
541+
return new Range(this.startLineNumber + deltaRange.startLineNumber, deltaRange.startColumn, this.startLineNumber + deltaRange.endLineNumber, deltaRange.endColumn);
542+
}
543+
544+
public removeTokens(range: Range): void {
545+
const startLineIndex = range.startLineNumber - this.startLineNumber;
546+
const endLineIndex = range.endLineNumber - this.startLineNumber;
547+
548+
this.startLineNumber += this.tokens.removeTokens(startLineIndex, range.startColumn - 1, endLineIndex, range.endColumn - 1);
549+
this._updateEndLineNumber();
550+
}
551+
479552
public applyEdit(range: IRange, text: string): void {
480553
const [eolCount, firstLineLength, lastLineLength] = countEOL(text);
481554
this.acceptEdit(range, eolCount, firstLineLength, lastLineLength, text.length > 0 ? text.charCodeAt(0) : CharCode.Null);
@@ -749,11 +822,50 @@ export class TokensStore2 {
749822
this._isComplete = false;
750823
}
751824

752-
public set(pieces: MultilineTokens2[] | null, isComplete: boolean) {
825+
public set(pieces: MultilineTokens2[] | null, isComplete: boolean): void {
753826
this._pieces = pieces || [];
754827
this._isComplete = isComplete;
755828
}
756829

830+
public setPartial(_range: Range, pieces: MultilineTokens2[]): Range {
831+
if (pieces.length === 0) {
832+
return _range;
833+
}
834+
const _firstRange = pieces[0].getRange();
835+
const _lastRange = pieces[pieces.length - 1].getRange();
836+
if (!_firstRange || !_lastRange) {
837+
return _range;
838+
}
839+
const range = _range.plusRange(_firstRange).plusRange(_lastRange);
840+
let insertIndex = this._pieces.length;
841+
for (let i = 0, len = this._pieces.length; i < len; i++) {
842+
const piece = this._pieces[i];
843+
if (piece.endLineNumber < range.startLineNumber) {
844+
continue;
845+
}
846+
if (piece.startLineNumber > range.endLineNumber) {
847+
insertIndex = Math.min(i, insertIndex);
848+
break;
849+
}
850+
piece.removeTokens(range);
851+
852+
if (piece.isEmpty()) {
853+
this._pieces.splice(i, 1);
854+
i--;
855+
len--;
856+
insertIndex--;
857+
continue;
858+
}
859+
860+
if (piece.startLineNumber >= range.endLineNumber) {
861+
insertIndex = Math.min(i, insertIndex);
862+
}
863+
}
864+
865+
this._pieces = arrays.arrayInsert(this._pieces, insertIndex, pieces);
866+
return range;
867+
}
868+
757869
public isComplete(): boolean {
758870
return this._isComplete;
759871
}
@@ -766,7 +878,7 @@ export class TokensStore2 {
766878
}
767879

768880
const pieceIndex = TokensStore2._findFirstPieceWithLine(pieces, lineNumber);
769-
const bTokens = this._pieces[pieceIndex].getLineTokens(lineNumber);
881+
const bTokens = pieces[pieceIndex].getLineTokens(lineNumber);
770882

771883
if (!bTokens) {
772884
return aTokens;

‎src/vs/editor/contrib/viewportSemanticTokens/viewportSemanticTokens.ts‎

Lines changed: 39 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -3,15 +3,16 @@
33
* Licensed under the MIT License. See License.txt in the project root for license information.
44
*--------------------------------------------------------------------------------------------*/
55

6-
import { RunOnceScheduler, createCancelablePromise } from 'vs/base/common/async';
6+
import { RunOnceScheduler, createCancelablePromise, CancelablePromise } from 'vs/base/common/async';
77
import { Disposable } from 'vs/base/common/lifecycle';
8+
import { Range } from 'vs/editor/common/core/range';
89
import { ICodeEditor } from 'vs/editor/browser/editorBrowser';
910
import { registerEditorContribution } from 'vs/editor/browser/editorExtensions';
1011
import { IEditorContribution } from 'vs/editor/common/editorCommon';
1112
import { ITextModel } from 'vs/editor/common/model';
12-
import { DocumentRangeSemanticTokensProviderRegistry, DocumentRangeSemanticTokensProvider } from 'vs/editor/common/modes';
13+
import { DocumentRangeSemanticTokensProviderRegistry, DocumentRangeSemanticTokensProvider, SemanticTokens } from 'vs/editor/common/modes';
1314
import { IModelService } from 'vs/editor/common/services/modelService';
14-
import { toMultilineTokens2 } from 'vs/editor/common/services/semanticTokensProviderStyling';
15+
import { toMultilineTokens2, SemanticTokensProviderStyling } from 'vs/editor/common/services/semanticTokensProviderStyling';
1516

1617
class ViewportSemanticTokensContribution extends Disposable implements IEditorContribution {
1718

@@ -23,6 +24,7 @@ class ViewportSemanticTokensContribution extends Disposable implements IEditorCo
2324

2425
private readonly _editor: ICodeEditor;
2526
private readonly _tokenizeViewport: RunOnceScheduler;
27+
private _outstandingRequests: CancelablePromise<SemanticTokens | null | undefined>[];
2628

2729
constructor(
2830
editor: ICodeEditor,
@@ -31,13 +33,20 @@ class ViewportSemanticTokensContribution extends Disposable implements IEditorCo
3133
super();
3234
this._editor = editor;
3335
this._tokenizeViewport = new RunOnceScheduler(() => this._tokenizeViewportNow(), 100);
36+
this._outstandingRequests = [];
3437
this._register(this._editor.onDidScrollChange(() => {
3538
this._tokenizeViewport.schedule();
3639
}));
3740
this._register(this._editor.onDidChangeModel(() => {
41+
this._cancelAll();
42+
this._tokenizeViewport.schedule();
43+
}));
44+
this._register(this._editor.onDidChangeModelContent((e) => {
45+
this._cancelAll();
3846
this._tokenizeViewport.schedule();
3947
}));
4048
this._register(DocumentRangeSemanticTokensProviderRegistry.onDidChange(() => {
49+
this._cancelAll();
4150
this._tokenizeViewport.schedule();
4251
}));
4352
}
@@ -47,6 +56,22 @@ class ViewportSemanticTokensContribution extends Disposable implements IEditorCo
4756
return (result.length > 0 ? result[0] : null);
4857
}
4958

59+
private _cancelAll(): void {
60+
for (const request of this._outstandingRequests) {
61+
request.cancel();
62+
}
63+
this._outstandingRequests = [];
64+
}
65+
66+
private _removeOutstandingRequest(req: CancelablePromise<SemanticTokens | null | undefined>): void {
67+
for (let i = 0, len = this._outstandingRequests.length; i < len; i++) {
68+
if (this._outstandingRequests[i] === req) {
69+
this._outstandingRequests.splice(i, 1);
70+
return;
71+
}
72+
}
73+
}
74+
5075
private _tokenizeViewportNow(): void {
5176
if (!this._editor.hasModel()) {
5277
return;
@@ -61,14 +86,20 @@ class ViewportSemanticTokensContribution extends Disposable implements IEditorCo
6186
}
6287
const styling = this._modelService.getSemanticTokensProviderStyling(provider);
6388
const visibleRanges = this._editor.getVisibleRanges();
64-
const request = createCancelablePromise(token => Promise.resolve(provider.provideDocumentRangeSemanticTokens(model, visibleRanges[0], token)));
89+
90+
this._outstandingRequests = this._outstandingRequests.concat(visibleRanges.map(range => this._requestRange(model, range, provider, styling)));
91+
}
92+
93+
private _requestRange(model: ITextModel, range: Range, provider: DocumentRangeSemanticTokensProvider, styling: SemanticTokensProviderStyling): CancelablePromise<SemanticTokens | null | undefined> {
94+
const requestVersionId = model.getVersionId();
95+
const request = createCancelablePromise(token => Promise.resolve(provider.provideDocumentRangeSemanticTokens(model, range, token)));
6596
request.then((r) => {
66-
if (!r || model.isDisposed()) {
97+
if (!r || model.isDisposed() || model.getVersionId() !== requestVersionId) {
6798
return;
6899
}
69-
const tokens = toMultilineTokens2(r, styling);
70-
model.setPartialSemanticTokens(tokens);
71-
});
100+
model.setPartialSemanticTokens(range, toMultilineTokens2(r, styling));
101+
}).then(() => this._removeOutstandingRequest(request), () => this._removeOutstandingRequest(request));
102+
return request;
72103
}
73104
}
74105

0 commit comments

Comments
 (0)