Skip to content

Commit 9e1d730

Browse files
committed
SemanticTokens - implement feedback received in API call:
- extract a separate DocumentRangeSemanticTokensProvider that deals with a document range - extract a separate provideDocumentSemanticTokensEdits that deals with updating via SemanticTokensEdits a previous result
1 parent 430de16 commit 9e1d730

12 files changed

Lines changed: 270 additions & 174 deletions

File tree

extensions/html-language-features/client/src/htmlMain.ts

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ const localize = nls.loadMessageBundle();
1111
import {
1212
languages, ExtensionContext, IndentAction, Position, TextDocument, Range, CompletionItem, CompletionItemKind, SnippetString, workspace,
1313
Disposable, FormattingOptions, CancellationToken, ProviderResult, TextEdit, CompletionContext, CompletionList, SemanticTokensLegend,
14-
SemanticTokensProvider, SemanticTokens
14+
DocumentSemanticTokensProvider, DocumentRangeSemanticTokensProvider, SemanticTokens
1515
} from 'vscode';
1616
import {
1717
LanguageClient, LanguageClientOptions, ServerOptions, TransportKind, RequestType, TextDocumentPositionParams, DocumentRangeFormattingParams,
@@ -153,18 +153,26 @@ export function activate(context: ExtensionContext) {
153153

154154
client.sendRequest(SemanticTokenLegendRequest.type).then(legend => {
155155
if (legend) {
156-
const provider: SemanticTokensProvider = {
157-
provideSemanticTokens(doc, opts) {
156+
const provider: DocumentSemanticTokensProvider & DocumentRangeSemanticTokensProvider = {
157+
provideDocumentSemanticTokens(doc) {
158158
const params: SemanticTokenParams = {
159159
textDocument: client.code2ProtocolConverter.asTextDocumentIdentifier(doc),
160-
ranges: opts.ranges?.map(r => client.code2ProtocolConverter.asRange(r))
160+
};
161+
return client.sendRequest(SemanticTokenRequest.type, params).then(data => {
162+
return data && new SemanticTokens(new Uint32Array(data));
163+
});
164+
},
165+
provideDocumentRangeSemanticTokens(doc, range) {
166+
const params: SemanticTokenParams = {
167+
textDocument: client.code2ProtocolConverter.asTextDocumentIdentifier(doc),
168+
ranges: [client.code2ProtocolConverter.asRange(range)]
161169
};
162170
return client.sendRequest(SemanticTokenRequest.type, params).then(data => {
163171
return data && new SemanticTokens(new Uint32Array(data));
164172
});
165173
}
166174
};
167-
toDispose.push(languages.registerSemanticTokensProvider(documentSelector, provider, new SemanticTokensLegend(legend.types, legend.modifiers)));
175+
toDispose.push(languages.registerDocumentSemanticTokensProvider(documentSelector, provider, new SemanticTokensLegend(legend.types, legend.modifiers)));
168176
}
169177
});
170178

extensions/typescript-language-features/src/features/semanticTokens.ts

Lines changed: 51 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -16,17 +16,20 @@ const minTypeScriptVersion = API.fromVersionString(`${VersionRequirement.major}.
1616

1717
export function register(selector: vscode.DocumentSelector, client: ITypeScriptServiceClient) {
1818
return new VersionDependentRegistration(client, minTypeScriptVersion, () => {
19-
const provider = new SemanticTokensProvider(client);
20-
return vscode.languages.registerSemanticTokensProvider(selector, provider, provider.getLegend());
19+
const provider = new DocumentSemanticTokensProvider(client);
20+
return vscode.Disposable.from(
21+
vscode.languages.registerDocumentSemanticTokensProvider(selector, provider, provider.getLegend()),
22+
vscode.languages.registerDocumentRangeSemanticTokensProvider(selector, provider, provider.getLegend()),
23+
);
2124
});
2225
}
2326

2427
/**
25-
* Prototype of a SemanticTokensProvider, relying on the experimental `encodedSemanticClassifications-full` request from the TypeScript server.
28+
* Prototype of a DocumentSemanticTokensProvider, relying on the experimental `encodedSemanticClassifications-full` request from the TypeScript server.
2629
* As the results retured by the TypeScript server are limited, we also add a Typescript plugin (typescript-vscode-sh-plugin) to enrich the returned token.
2730
* See https://github.com/aeschli/typescript-vscode-sh-plugin.
2831
*/
29-
class SemanticTokensProvider implements vscode.SemanticTokensProvider {
32+
class DocumentSemanticTokensProvider implements vscode.DocumentSemanticTokensProvider, vscode.DocumentRangeSemanticTokensProvider {
3033

3134
constructor(private readonly client: ITypeScriptServiceClient) {
3235
}
@@ -41,68 +44,65 @@ class SemanticTokensProvider implements vscode.SemanticTokensProvider {
4144
return new vscode.SemanticTokensLegend(tokenTypes, tokenModifiers);
4245
}
4346

44-
async provideSemanticTokens(document: vscode.TextDocument, options: vscode.SemanticTokensRequestOptions, token: vscode.CancellationToken): Promise<vscode.SemanticTokens | null> {
47+
async provideDocumentSemanticTokens(document: vscode.TextDocument, token: vscode.CancellationToken): Promise<vscode.SemanticTokens | null> {
4548
const file = this.client.toOpenedFilePath(document);
4649
if (!file) {
4750
return null;
4851
}
52+
return this._provideSemanticTokens(document, { file, start: 0, length: document.getText().length }, token);
53+
}
4954

50-
const versionBeforeRequest = document.version;
51-
52-
const allTokenSpans: number[][] = [];
53-
54-
let requestArgs: ExperimentalProtocol.EncodedSemanticClassificationsRequestArgs[] = [];
55-
if (options.ranges) {
56-
requestArgs = options.ranges.map(r => { const start = document.offsetAt(r.start); const length = document.offsetAt(r.end) - start; return { file, start, length }; });
57-
requestArgs = requestArgs.sort((a1, a2) => a1.start - a2.start);
58-
} else {
59-
requestArgs = [{ file, start: 0, length: document.getText().length }]; // full document
55+
async provideDocumentRangeSemanticTokens(document: vscode.TextDocument, range: vscode.Range, token: vscode.CancellationToken): Promise<vscode.SemanticTokens | null> {
56+
const file = this.client.toOpenedFilePath(document);
57+
if (!file) {
58+
return null;
6059
}
61-
for (const requestArg of requestArgs) {
62-
const response = await (this.client as ExperimentalProtocol.IExtendedTypeScriptServiceClient).execute('encodedSemanticClassifications-full', requestArg, token);
63-
if (response.type === 'response' && response.body) {
64-
allTokenSpans.push(response.body.spans);
65-
} else {
66-
return null;
67-
}
60+
const start = document.offsetAt(range.start);
61+
const length = document.offsetAt(range.end) - start;
62+
return this._provideSemanticTokens(document, { file, start, length }, token);
63+
}
64+
65+
async _provideSemanticTokens(document: vscode.TextDocument, requestArg: ExperimentalProtocol.EncodedSemanticClassificationsRequestArgs, token: vscode.CancellationToken): Promise<vscode.SemanticTokens | null> {
66+
const file = this.client.toOpenedFilePath(document);
67+
if (!file) {
68+
return null;
6869
}
6970

70-
const versionAfterRequest = document.version;
71-
if (versionBeforeRequest !== versionAfterRequest) {
72-
// A new request will come in soon...
71+
const response = await (this.client as ExperimentalProtocol.IExtendedTypeScriptServiceClient).execute('encodedSemanticClassifications-full', requestArg, token);
72+
if (response.type !== 'response' || !response.body) {
7373
return null;
7474
}
7575

76+
const tokenSpan = response.body.spans;
77+
7678
const builder = new vscode.SemanticTokensBuilder();
77-
for (const tokenSpan of allTokenSpans) {
78-
let i = 0;
79-
while (i < tokenSpan.length) {
80-
const offset = tokenSpan[i++];
81-
const length = tokenSpan[i++];
82-
const tsClassification = tokenSpan[i++];
83-
84-
let tokenModifiers = 0;
85-
let tokenType = getTokenTypeFromClassification(tsClassification);
86-
if (tokenType !== undefined) {
87-
// it's a classification as returned by the typescript-vscode-sh-plugin
88-
tokenModifiers = getTokenModifierFromClassification(tsClassification);
89-
} else {
90-
// typescript-vscode-sh-plugin is not present
91-
tokenType = tokenTypeMap[tsClassification];
92-
if (tokenType === undefined) {
93-
continue;
94-
}
79+
let i = 0;
80+
while (i < tokenSpan.length) {
81+
const offset = tokenSpan[i++];
82+
const length = tokenSpan[i++];
83+
const tsClassification = tokenSpan[i++];
84+
85+
let tokenModifiers = 0;
86+
let tokenType = getTokenTypeFromClassification(tsClassification);
87+
if (tokenType !== undefined) {
88+
// it's a classification as returned by the typescript-vscode-sh-plugin
89+
tokenModifiers = getTokenModifierFromClassification(tsClassification);
90+
} else {
91+
// typescript-vscode-sh-plugin is not present
92+
tokenType = tokenTypeMap[tsClassification];
93+
if (tokenType === undefined) {
94+
continue;
9595
}
96+
}
9697

97-
// we can use the document's range conversion methods because the result is at the same version as the document
98-
const startPos = document.positionAt(offset);
99-
const endPos = document.positionAt(offset + length);
98+
// we can use the document's range conversion methods because the result is at the same version as the document
99+
const startPos = document.positionAt(offset);
100+
const endPos = document.positionAt(offset + length);
100101

101-
for (let line = startPos.line; line <= endPos.line; line++) {
102-
const startCharacter = (line === startPos.line ? startPos.character : 0);
103-
const endCharacter = (line === endPos.line ? endPos.character : document.lineAt(line).text.length);
104-
builder.push(line, startCharacter, endCharacter - startCharacter, tokenType, tokenModifiers);
105-
}
102+
for (let line = startPos.line; line <= endPos.line; line++) {
103+
const startCharacter = (line === startPos.line ? startPos.character : 0);
104+
const endCharacter = (line === endPos.line ? endPos.character : document.lineAt(line).text.length);
105+
builder.push(line, startCharacter, endCharacter - startCharacter, tokenType, tokenModifiers);
106106
}
107107
}
108108
return new vscode.SemanticTokens(builder.build());

extensions/vscode-colorize-tests/src/colorizerTestMain.ts

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,8 @@ export function activate(context: vscode.ExtensionContext): any {
1515

1616
const outputChannel = vscode.window.createOutputChannel('Semantic Tokens Test');
1717

18-
const semanticHighlightProvider: vscode.SemanticTokensProvider = {
19-
provideSemanticTokens(document: vscode.TextDocument): vscode.ProviderResult<vscode.SemanticTokens> {
18+
const documentSemanticHighlightProvider: vscode.DocumentSemanticTokensProvider = {
19+
provideDocumentSemanticTokens(document: vscode.TextDocument): vscode.ProviderResult<vscode.SemanticTokens> {
2020
const builder = new vscode.SemanticTokensBuilder();
2121

2222
function addToken(value: string, startLine: number, startCharacter: number, length: number) {
@@ -61,6 +61,6 @@ export function activate(context: vscode.ExtensionContext): any {
6161
};
6262

6363

64-
context.subscriptions.push(vscode.languages.registerSemanticTokensProvider({ pattern: '**/*semantic-test.json' }, semanticHighlightProvider, legend));
64+
context.subscriptions.push(vscode.languages.registerDocumentSemanticTokensProvider({ pattern: '**/*semantic-test.json' }, documentSemanticHighlightProvider, legend));
6565

6666
}

src/vs/editor/common/modes.ts

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1500,10 +1500,15 @@ export interface SemanticTokensEdits {
15001500
readonly edits: SemanticTokensEdit[];
15011501
}
15021502

1503-
export interface SemanticTokensProvider {
1503+
export interface DocumentSemanticTokensProvider {
15041504
getLegend(): SemanticTokensLegend;
1505-
provideSemanticTokens(model: model.ITextModel, lastResultId: string | null, ranges: Range[] | null, token: CancellationToken): ProviderResult<SemanticTokens | SemanticTokensEdits>;
1506-
releaseSemanticTokens(resultId: string | undefined): void;
1505+
provideDocumentSemanticTokens(model: model.ITextModel, lastResultId: string | null, token: CancellationToken): ProviderResult<SemanticTokens | SemanticTokensEdits>;
1506+
releaseDocumentSemanticTokens(resultId: string | undefined): void;
1507+
}
1508+
1509+
export interface DocumentRangeSemanticTokensProvider {
1510+
getLegend(): SemanticTokensLegend;
1511+
provideDocumentRangeSemanticTokens(model: model.ITextModel, range: Range, token: CancellationToken): ProviderResult<SemanticTokens>;
15071512
}
15081513

15091514
// --- feature registries ------
@@ -1611,7 +1616,12 @@ export const FoldingRangeProviderRegistry = new LanguageFeatureRegistry<FoldingR
16111616
/**
16121617
* @internal
16131618
*/
1614-
export const SemanticTokensProviderRegistry = new LanguageFeatureRegistry<SemanticTokensProvider>();
1619+
export const DocumentSemanticTokensProviderRegistry = new LanguageFeatureRegistry<DocumentSemanticTokensProvider>();
1620+
1621+
/**
1622+
* @internal
1623+
*/
1624+
export const DocumentRangeSemanticTokensProviderRegistry = new LanguageFeatureRegistry<DocumentRangeSemanticTokensProvider>();
16151625

16161626
/**
16171627
* @internal

src/vs/editor/common/services/modelServiceImpl.ts

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ import { Range } from 'vs/editor/common/core/range';
1414
import { DefaultEndOfLine, EndOfLinePreference, EndOfLineSequence, IIdentifiedSingleEditOperation, ITextBuffer, ITextBufferFactory, ITextModel, ITextModelCreationOptions } from 'vs/editor/common/model';
1515
import { TextModel, createTextBuffer } from 'vs/editor/common/model/textModel';
1616
import { IModelLanguageChangedEvent, IModelContentChangedEvent } from 'vs/editor/common/model/textModelEvents';
17-
import { LanguageIdentifier, SemanticTokensProviderRegistry, SemanticTokensProvider, SemanticTokensLegend, SemanticTokens, SemanticTokensEdits, TokenMetadata } from 'vs/editor/common/modes';
17+
import { LanguageIdentifier, DocumentSemanticTokensProviderRegistry, DocumentSemanticTokensProvider, SemanticTokensLegend, SemanticTokens, SemanticTokensEdits, TokenMetadata } from 'vs/editor/common/modes';
1818
import { PLAINTEXT_LANGUAGE_IDENTIFIER } from 'vs/editor/common/modes/modesRegistry';
1919
import { ILanguageSelection } from 'vs/editor/common/services/modeService';
2020
import { IModelService } from 'vs/editor/common/services/modelService';
@@ -498,23 +498,23 @@ class SemanticColoringFeature extends Disposable {
498498

499499
class SemanticStyling extends Disposable {
500500

501-
private _caches: WeakMap<SemanticTokensProvider, SemanticColoringProviderStyling>;
501+
private _caches: WeakMap<DocumentSemanticTokensProvider, SemanticColoringProviderStyling>;
502502

503503
constructor(
504504
private readonly _themeService: IThemeService,
505505
private readonly _logService: ILogService
506506
) {
507507
super();
508-
this._caches = new WeakMap<SemanticTokensProvider, SemanticColoringProviderStyling>();
508+
this._caches = new WeakMap<DocumentSemanticTokensProvider, SemanticColoringProviderStyling>();
509509
if (this._themeService) {
510510
// workaround for tests which use undefined... :/
511511
this._register(this._themeService.onThemeChange(() => {
512-
this._caches = new WeakMap<SemanticTokensProvider, SemanticColoringProviderStyling>();
512+
this._caches = new WeakMap<DocumentSemanticTokensProvider, SemanticColoringProviderStyling>();
513513
}));
514514
}
515515
}
516516

517-
public get(provider: SemanticTokensProvider): SemanticColoringProviderStyling {
517+
public get(provider: DocumentSemanticTokensProvider): SemanticColoringProviderStyling {
518518
if (!this._caches.has(provider)) {
519519
this._caches.set(provider, new SemanticColoringProviderStyling(provider.getLegend(), this._themeService, this._logService));
520520
}
@@ -676,13 +676,13 @@ const enum SemanticColoringConstants {
676676

677677
class SemanticTokensResponse {
678678
constructor(
679-
private readonly _provider: SemanticTokensProvider,
679+
private readonly _provider: DocumentSemanticTokensProvider,
680680
public readonly resultId: string | undefined,
681681
public readonly data: Uint32Array
682682
) { }
683683

684684
public dispose(): void {
685-
this._provider.releaseSemanticTokens(this.resultId);
685+
this._provider.releaseDocumentSemanticTokens(this.resultId);
686686
}
687687
}
688688

@@ -710,7 +710,7 @@ class ModelSemanticColoring extends Disposable {
710710
this._fetchSemanticTokens.schedule();
711711
}
712712
}));
713-
this._register(SemanticTokensProviderRegistry.onDidChange(e => this._fetchSemanticTokens.schedule()));
713+
this._register(DocumentSemanticTokensProviderRegistry.onDidChange(e => this._fetchSemanticTokens.schedule()));
714714
if (themeService) {
715715
// workaround for tests which use undefined... :/
716716
this._register(themeService.onThemeChange(_ => {
@@ -756,7 +756,7 @@ class ModelSemanticColoring extends Disposable {
756756
const styling = this._semanticStyling.get(provider);
757757

758758
const lastResultId = this._currentResponse ? this._currentResponse.resultId || null : null;
759-
const request = Promise.resolve(provider.provideSemanticTokens(this._model, lastResultId, null, this._currentRequestCancellationTokenSource.token));
759+
const request = Promise.resolve(provider.provideDocumentSemanticTokens(this._model, lastResultId, this._currentRequestCancellationTokenSource.token));
760760

761761
request.then((res) => {
762762
this._currentRequestCancellationTokenSource = null;
@@ -784,7 +784,7 @@ class ModelSemanticColoring extends Disposable {
784784
}
785785
}
786786

787-
private _setSemanticTokens(provider: SemanticTokensProvider | null, tokens: SemanticTokens | SemanticTokensEdits | null, styling: SemanticColoringProviderStyling | null, pendingChanges: IModelContentChangedEvent[]): void {
787+
private _setSemanticTokens(provider: DocumentSemanticTokensProvider | null, tokens: SemanticTokens | SemanticTokensEdits | null, styling: SemanticColoringProviderStyling | null, pendingChanges: IModelContentChangedEvent[]): void {
788788
const currentResponse = this._currentResponse;
789789
if (this._currentResponse) {
790790
this._currentResponse.dispose();
@@ -793,7 +793,7 @@ class ModelSemanticColoring extends Disposable {
793793
if (this._isDisposed) {
794794
// disposed!
795795
if (provider && tokens) {
796-
provider.releaseSemanticTokens(tokens.resultId);
796+
provider.releaseDocumentSemanticTokens(tokens.resultId);
797797
}
798798
return;
799799
}
@@ -954,8 +954,8 @@ class ModelSemanticColoring extends Disposable {
954954
this._model.setSemanticTokens(null);
955955
}
956956

957-
private _getSemanticColoringProvider(): SemanticTokensProvider | null {
958-
const result = SemanticTokensProviderRegistry.ordered(this._model);
957+
private _getSemanticColoringProvider(): DocumentSemanticTokensProvider | null {
958+
const result = DocumentSemanticTokensProviderRegistry.ordered(this._model);
959959
return (result.length > 0 ? result[0] : null);
960960
}
961961
}

src/vs/monaco.d.ts

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5948,10 +5948,15 @@ declare namespace monaco.languages {
59485948
readonly edits: SemanticTokensEdit[];
59495949
}
59505950

5951-
export interface SemanticTokensProvider {
5951+
export interface DocumentSemanticTokensProvider {
59525952
getLegend(): SemanticTokensLegend;
5953-
provideSemanticTokens(model: editor.ITextModel, lastResultId: string | null, ranges: Range[] | null, token: CancellationToken): ProviderResult<SemanticTokens | SemanticTokensEdits>;
5954-
releaseSemanticTokens(resultId: string | undefined): void;
5953+
provideDocumentSemanticTokens(model: editor.ITextModel, lastResultId: string | null, token: CancellationToken): ProviderResult<SemanticTokens | SemanticTokensEdits>;
5954+
releaseDocumentSemanticTokens(resultId: string | undefined): void;
5955+
}
5956+
5957+
export interface DocumentRangeSemanticTokensProvider {
5958+
getLegend(): SemanticTokensLegend;
5959+
provideDocumentRangeSemanticTokens(model: editor.ITextModel, range: Range, token: CancellationToken): ProviderResult<SemanticTokens>;
59555960
}
59565961

59575962
export interface ILanguageExtensionPoint {

0 commit comments

Comments
 (0)