Skip to content

Commit 0134339

Browse files
alexdimaalexdima
committed
Semantic tokenization
1 parent 308ff31 commit 0134339

8 files changed

Lines changed: 410 additions & 78 deletions

File tree

‎extensions/typescript-language-features/src/features/semanticColoring.ts‎

Lines changed: 1 addition & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -112,47 +112,5 @@ export function register(
112112
selector: vscode.DocumentSelector,
113113
client: ITypeScriptServiceClient
114114
) {
115-
const provider = new SemanticColoringProvider(client);
116-
117-
118-
const run = async () => {
119-
const ed = vscode.window.activeTextEditor;
120-
if (!ed) {
121-
return;
122-
}
123-
// const doc = ed.document;
124-
const cancellationTokenSource = new vscode.CancellationTokenSource();
125-
provider.provideSemanticColoring(ed.document, cancellationTokenSource.token);
126-
// const file = client.toOpenedFilePath(doc);
127-
// if (!file) {
128-
// return;
129-
// }
130-
// const args: ExperimentalProtocol.EncodedSemanticClassificationsRequestArgs = {
131-
// file: file,
132-
// start: 0,
133-
// length: doc.getText().length,
134-
// };
135-
136-
// const response = await client.execute('encodedSemanticClassifications-full', args, cancellationTokenSource.token);
137-
138-
// if (response.type !== 'response') {
139-
// return;
140-
// }
141-
// if (!response.body) {
142-
// return;
143-
// }
144-
// console.log(response.body);
145-
};
146-
147-
vscode.window.onDidChangeActiveTextEditor(run);
148-
run();
149-
150-
console.log(`I am running...`);
151-
152-
// return vscode.Disposable.from();
153-
// return vscode.languages.registerRenameProvider(selector,
154-
// new TypeScriptRenameProvider(client, fileConfigurationManager));
155-
156-
return vscode.languages.registerSemanticColoringProvider(selector, provider);
157-
// return vscode.languages.registerSemanticColoringProvider(selector, )
115+
return vscode.languages.registerSemanticColoringProvider(selector, new SemanticColoringProvider(client));
158116
}

‎src/vs/editor/common/core/lineTokens.ts‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,11 @@ export class LineTokens implements IViewLineTokens {
6767
return 0;
6868
}
6969

70+
public getMetadata(tokenIndex: number): number {
71+
const metadata = this._tokens[(tokenIndex << 1) + 1];
72+
return metadata;
73+
}
74+
7075
public getLanguageId(tokenIndex: number): LanguageId {
7176
const metadata = this._tokens[(tokenIndex << 1) + 1];
7277
return TokenMetadata.getLanguageId(metadata);
@@ -132,8 +137,8 @@ export class LineTokens implements IViewLineTokens {
132137

133138
while (low < high) {
134139

135-
let mid = low + Math.floor((high - low) / 2);
136-
let endOffset = tokens[(mid << 1)];
140+
const mid = low + Math.floor((high - low) / 2);
141+
const endOffset = tokens[(mid << 1)];
137142

138143
if (endOffset === desiredIndex) {
139144
return mid + 1;

‎src/vs/editor/common/model.ts‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ import { IModelContentChange, IModelContentChangedEvent, IModelDecorationsChange
1414
import { SearchData } from 'vs/editor/common/model/textModelSearch';
1515
import { LanguageId, LanguageIdentifier, FormattingOptions } from 'vs/editor/common/modes';
1616
import { ThemeColor } from 'vs/platform/theme/common/themeService';
17-
import { MultilineTokens } from 'vs/editor/common/model/tokensStore';
17+
import { MultilineTokens, MultilineTokens2 } from 'vs/editor/common/model/tokensStore';
1818

1919
/**
2020
* Vertical Lane in the overview ruler of the editor.
@@ -791,6 +791,11 @@ export interface ITextModel {
791791
*/
792792
setTokens(tokens: MultilineTokens[]): void;
793793

794+
/**
795+
* @internal
796+
*/
797+
setSemanticTokens(tokens: MultilineTokens2[]): void;
798+
794799
/**
795800
* Flush all tokenization state.
796801
* @internal

‎src/vs/editor/common/model/textModel.ts‎

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ import { BracketsUtils, RichEditBracket, RichEditBrackets } from 'vs/editor/comm
3232
import { ITheme, ThemeColor } from 'vs/platform/theme/common/themeService';
3333
import { withUndefinedAsNull } from 'vs/base/common/types';
3434
import { VSBufferReadableStream, VSBuffer } from 'vs/base/common/buffer';
35-
import { TokensStore, MultilineTokens, countEOL } from 'vs/editor/common/model/tokensStore';
35+
import { TokensStore, MultilineTokens, countEOL, MultilineTokens2, TokensStore2 } from 'vs/editor/common/model/tokensStore';
3636
import { Color } from 'vs/base/common/color';
3737

3838
function createTextBufferBuilder() {
@@ -276,7 +276,7 @@ export class TextModel extends Disposable implements model.ITextModel {
276276
private _languageIdentifier: LanguageIdentifier;
277277
private readonly _languageRegistryListener: IDisposable;
278278
private readonly _tokens: TokensStore;
279-
private readonly _tokens2: TokensStore;
279+
private readonly _tokens2: TokensStore2;
280280
private readonly _tokenization: TextModelTokenization;
281281
//#endregion
282282

@@ -340,7 +340,7 @@ export class TextModel extends Disposable implements model.ITextModel {
340340
this._trimAutoWhitespaceLines = null;
341341

342342
this._tokens = new TokensStore();
343-
this._tokens2 = new TokensStore();
343+
this._tokens2 = new TokensStore2();
344344
this._tokenization = new TextModelTokenization(this);
345345
}
346346

@@ -1721,8 +1721,14 @@ export class TextModel extends Disposable implements model.ITextModel {
17211721
});
17221722
}
17231723

1724-
public setSemanticTokens(tokens: MultilineTokens[]): void {
1725-
1724+
public setSemanticTokens(tokens: MultilineTokens2[]): void {
1725+
this._tokens2.set(tokens);
1726+
1727+
// TODO@semantic: could we reduce the event here?
1728+
this._emitModelTokensChangedEvent({
1729+
tokenizationSupportChanged: false,
1730+
ranges: [{ fromLineNumber: 1, toLineNumber: this.getLineCount() }]
1731+
});
17261732
}
17271733

17281734
public tokenizeViewport(startLineNumber: number, endLineNumber: number): void {
@@ -1785,7 +1791,8 @@ export class TextModel extends Disposable implements model.ITextModel {
17851791

17861792
private _getLineTokens(lineNumber: number): LineTokens {
17871793
const lineText = this.getLineContent(lineNumber);
1788-
return this._tokens.getTokens(this._languageIdentifier.id, lineNumber - 1, lineText);
1794+
const syntacticTokens = this._tokens.getTokens(this._languageIdentifier.id, lineNumber - 1, lineText);
1795+
return this._tokens2.addSemanticTokens(lineNumber, syntacticTokens);
17891796
}
17901797

17911798
public getLanguageIdentifier(): LanguageIdentifier {

0 commit comments

Comments
 (0)