Skip to content

Commit ca0adce

Browse files
alexdimaalexdima
authored
Semantic exploration (microsoft#84788)
Semantic exploration
2 parents 1c181b4 + e7f8a94 commit ca0adce

16 files changed

Lines changed: 1876 additions & 14 deletions

File tree

‎extensions/typescript-language-features/src/languageProvider.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -141,4 +141,4 @@ export default class LanguageProvider extends Disposable {
141141
private get _diagnosticLanguage() {
142142
return this.description.diagnosticLanguage;
143143
}
144-
}
144+
}

‎src/vs/editor/common/core/lineTokens.ts‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,11 @@ export class LineTokens implements IViewLineTokens {
6767
return 0;
6868
}
6969

70+
public getMetadata(tokenIndex: number): number {
71+
const metadata = this._tokens[(tokenIndex << 1) + 1];
72+
return metadata;
73+
}
74+
7075
public getLanguageId(tokenIndex: number): LanguageId {
7176
const metadata = this._tokens[(tokenIndex << 1) + 1];
7277
return TokenMetadata.getLanguageId(metadata);
@@ -132,8 +137,8 @@ export class LineTokens implements IViewLineTokens {
132137

133138
while (low < high) {
134139

135-
let mid = low + Math.floor((high - low) / 2);
136-
let endOffset = tokens[(mid << 1)];
140+
const mid = low + Math.floor((high - low) / 2);
141+
const endOffset = tokens[(mid << 1)];
137142

138143
if (endOffset === desiredIndex) {
139144
return mid + 1;

‎src/vs/editor/common/model.ts‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ import { IModelContentChange, IModelContentChangedEvent, IModelDecorationsChange
1414
import { SearchData } from 'vs/editor/common/model/textModelSearch';
1515
import { LanguageId, LanguageIdentifier, FormattingOptions } from 'vs/editor/common/modes';
1616
import { ThemeColor } from 'vs/platform/theme/common/themeService';
17-
import { MultilineTokens } from 'vs/editor/common/model/tokensStore';
17+
import { MultilineTokens, MultilineTokens2 } from 'vs/editor/common/model/tokensStore';
1818

1919
/**
2020
* Vertical Lane in the overview ruler of the editor.
@@ -792,6 +792,11 @@ export interface ITextModel {
792792
*/
793793
setTokens(tokens: MultilineTokens[]): void;
794794

795+
/**
796+
* @internal
797+
*/
798+
setSemanticTokens(tokens: MultilineTokens2[] | null): void;
799+
795800
/**
796801
* Flush all tokenization state.
797802
* @internal

‎src/vs/editor/common/model/textModel.ts‎

Lines changed: 26 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ import { BracketsUtils, RichEditBracket, RichEditBrackets } from 'vs/editor/comm
3232
import { ITheme, ThemeColor } from 'vs/platform/theme/common/themeService';
3333
import { withUndefinedAsNull } from 'vs/base/common/types';
3434
import { VSBufferReadableStream, VSBuffer } from 'vs/base/common/buffer';
35-
import { TokensStore, MultilineTokens, countEOL } from 'vs/editor/common/model/tokensStore';
35+
import { TokensStore, MultilineTokens, countEOL, MultilineTokens2, TokensStore2 } from 'vs/editor/common/model/tokensStore';
3636
import { Color } from 'vs/base/common/color';
3737

3838
function createTextBufferBuilder() {
@@ -276,6 +276,7 @@ export class TextModel extends Disposable implements model.ITextModel {
276276
private _languageIdentifier: LanguageIdentifier;
277277
private readonly _languageRegistryListener: IDisposable;
278278
private readonly _tokens: TokensStore;
279+
private readonly _tokens2: TokensStore2;
279280
private readonly _tokenization: TextModelTokenization;
280281
//#endregion
281282

@@ -339,6 +340,7 @@ export class TextModel extends Disposable implements model.ITextModel {
339340
this._trimAutoWhitespaceLines = null;
340341

341342
this._tokens = new TokensStore();
343+
this._tokens2 = new TokensStore2();
342344
this._tokenization = new TextModelTokenization(this);
343345
}
344346

@@ -414,6 +416,7 @@ export class TextModel extends Disposable implements model.ITextModel {
414416

415417
// Flush all tokens
416418
this._tokens.flush();
419+
this._tokens2.flush();
417420

418421
// Destroy all my decorations
419422
this._decorations = Object.create(null);
@@ -1262,8 +1265,9 @@ export class TextModel extends Disposable implements model.ITextModel {
12621265
let lineCount = oldLineCount;
12631266
for (let i = 0, len = contentChanges.length; i < len; i++) {
12641267
const change = contentChanges[i];
1265-
const [eolCount, firstLineLength] = countEOL(change.text);
1268+
const [eolCount, firstLineLength, lastLineLength] = countEOL(change.text);
12661269
this._tokens.acceptEdit(change.range, eolCount, firstLineLength);
1270+
this._tokens2.acceptEdit(change.range, eolCount, firstLineLength, lastLineLength, change.text.length > 0 ? change.text.charCodeAt(0) : CharCode.Null);
12671271
this._onDidChangeDecorations.fire();
12681272
this._decorationsTree.acceptReplace(change.rangeOffset, change.rangeLength, change.text.length, change.forceMoveMarkers);
12691273

@@ -1717,6 +1721,15 @@ export class TextModel extends Disposable implements model.ITextModel {
17171721
});
17181722
}
17191723

1724+
public setSemanticTokens(tokens: MultilineTokens2[] | null): void {
1725+
this._tokens2.set(tokens);
1726+
1727+
this._emitModelTokensChangedEvent({
1728+
tokenizationSupportChanged: false,
1729+
ranges: [{ fromLineNumber: 1, toLineNumber: this.getLineCount() }]
1730+
});
1731+
}
1732+
17201733
public tokenizeViewport(startLineNumber: number, endLineNumber: number): void {
17211734
startLineNumber = Math.max(1, startLineNumber);
17221735
endLineNumber = Math.min(this._buffer.getLineCount(), endLineNumber);
@@ -1734,6 +1747,15 @@ export class TextModel extends Disposable implements model.ITextModel {
17341747
});
17351748
}
17361749

1750+
public clearSemanticTokens(): void {
1751+
this._tokens2.flush();
1752+
1753+
this._emitModelTokensChangedEvent({
1754+
tokenizationSupportChanged: false,
1755+
ranges: [{ fromLineNumber: 1, toLineNumber: this.getLineCount() }]
1756+
});
1757+
}
1758+
17371759
private _emitModelTokensChangedEvent(e: IModelTokensChangedEvent): void {
17381760
if (!this._isDisposing) {
17391761
this._onDidChangeTokens.fire(e);
@@ -1772,7 +1794,8 @@ export class TextModel extends Disposable implements model.ITextModel {
17721794

17731795
private _getLineTokens(lineNumber: number): LineTokens {
17741796
const lineText = this.getLineContent(lineNumber);
1775-
return this._tokens.getTokens(this._languageIdentifier.id, lineNumber - 1, lineText);
1797+
const syntacticTokens = this._tokens.getTokens(this._languageIdentifier.id, lineNumber - 1, lineText);
1798+
return this._tokens2.addSemanticTokens(lineNumber, syntacticTokens);
17761799
}
17771800

17781801
public getLanguageIdentifier(): LanguageIdentifier {

0 commit comments

Comments
 (0)