Skip to content

Commit 55646f7

Browse files
aeschliaeschli
authored
Merge pull request microsoft#93378 from microsoft/aeschli/tokenLanguage
language in token
2 parents 2c4c755 + 0cf78e1 commit 55646f7

10 files changed

Lines changed: 171 additions & 82 deletions

File tree

‎src/vs/editor/common/services/modelServiceImpl.ts‎

Lines changed: 19 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -661,12 +661,14 @@ const enum Constants {
661661
class HashTableEntry {
662662
public readonly tokenTypeIndex: number;
663663
public readonly tokenModifierSet: number;
664+
public readonly languageId: number;
664665
public readonly metadata: number;
665666
public next: HashTableEntry | null;
666667

667-
constructor(tokenTypeIndex: number, tokenModifierSet: number, metadata: number) {
668+
constructor(tokenTypeIndex: number, tokenModifierSet: number, languageId: number, metadata: number) {
668669
this.tokenTypeIndex = tokenTypeIndex;
669670
this.tokenModifierSet = tokenModifierSet;
671+
this.languageId = languageId;
670672
this.metadata = metadata;
671673
this.next = null;
672674
}
@@ -697,16 +699,17 @@ class HashTable {
697699
}
698700
}
699701

700-
private _hashFunc(tokenTypeIndex: number, tokenModifierSet: number): number {
701-
return ((((tokenTypeIndex << 5) - tokenTypeIndex) + tokenModifierSet) | 0) % this._currentLength; // tokenTypeIndex * 31 + tokenModifierSet, keep as int32
702+
private _hashFunc(tokenTypeIndex: number, tokenModifierSet: number, languageId: number): number {
703+
const hash = (n1: number, n2: number) => (((n1 << 5) - n1) + n2) | 0; // n1 * 31 + n2, keep as int32
704+
return hash(hash(tokenTypeIndex, tokenModifierSet), languageId) % this._currentLength;
702705
}
703706

704-
public get(tokenTypeIndex: number, tokenModifierSet: number): HashTableEntry | null {
705-
const hash = this._hashFunc(tokenTypeIndex, tokenModifierSet);
707+
public get(tokenTypeIndex: number, tokenModifierSet: number, languageId: number): HashTableEntry | null {
708+
const hash = this._hashFunc(tokenTypeIndex, tokenModifierSet, languageId);
706709

707710
let p = this._elements[hash];
708711
while (p) {
709-
if (p.tokenTypeIndex === tokenTypeIndex && p.tokenModifierSet === tokenModifierSet) {
712+
if (p.tokenTypeIndex === tokenTypeIndex && p.tokenModifierSet === tokenModifierSet && p.languageId === languageId) {
710713
return p;
711714
}
712715
p = p.next;
@@ -715,7 +718,7 @@ class HashTable {
715718
return null;
716719
}
717720

718-
public add(tokenTypeIndex: number, tokenModifierSet: number, metadata: number): void {
721+
public add(tokenTypeIndex: number, tokenModifierSet: number, languageId: number, metadata: number): void {
719722
this._elementsCount++;
720723
if (this._growCount !== 0 && this._elementsCount >= this._growCount) {
721724
// expand!
@@ -737,11 +740,11 @@ class HashTable {
737740
}
738741
}
739742
}
740-
this._add(new HashTableEntry(tokenTypeIndex, tokenModifierSet, metadata));
743+
this._add(new HashTableEntry(tokenTypeIndex, tokenModifierSet, languageId, metadata));
741744
}
742745

743746
private _add(element: HashTableEntry): void {
744-
const hash = this._hashFunc(element.tokenTypeIndex, element.tokenModifierSet);
747+
const hash = this._hashFunc(element.tokenTypeIndex, element.tokenModifierSet, element.languageId);
745748
element.next = this._elements[hash];
746749
this._elements[hash] = element;
747750
}
@@ -759,8 +762,8 @@ class SemanticColoringProviderStyling {
759762
this._hashTable = new HashTable();
760763
}
761764

762-
public getMetadata(tokenTypeIndex: number, tokenModifierSet: number): number {
763-
const entry = this._hashTable.get(tokenTypeIndex, tokenModifierSet);
765+
public getMetadata(tokenTypeIndex: number, tokenModifierSet: number, languageId: LanguageIdentifier): number {
766+
const entry = this._hashTable.get(tokenTypeIndex, tokenModifierSet, languageId.id);
764767
let metadata: number;
765768
if (entry) {
766769
metadata = entry.metadata;
@@ -775,7 +778,7 @@ class SemanticColoringProviderStyling {
775778
modifierSet = modifierSet >> 1;
776779
}
777780

778-
const tokenStyle = this._themeService.getColorTheme().getTokenStyleMetadata(tokenType, tokenModifiers);
781+
const tokenStyle = this._themeService.getColorTheme().getTokenStyleMetadata(tokenType, tokenModifiers, languageId.language);
779782
if (typeof tokenStyle === 'undefined') {
780783
metadata = Constants.NO_STYLING;
781784
} else {
@@ -801,7 +804,7 @@ class SemanticColoringProviderStyling {
801804
metadata = Constants.NO_STYLING;
802805
}
803806
}
804-
this._hashTable.add(tokenTypeIndex, tokenModifierSet, metadata);
807+
this._hashTable.add(tokenTypeIndex, tokenModifierSet, languageId.id, metadata);
805808
}
806809
if (this._logService.getLevel() === LogLevel.Trace) {
807810
const type = this._legend.tokenTypes[tokenTypeIndex];
@@ -1042,6 +1045,8 @@ class ModelSemanticColoring extends Disposable {
10421045

10431046
const result: MultilineTokens2[] = [];
10441047

1048+
const languageId = this._model.getLanguageIdentifier();
1049+
10451050
let tokenIndex = 0;
10461051
let lastLineNumber = 1;
10471052
let lastStartCharacter = 0;
@@ -1081,7 +1086,7 @@ class ModelSemanticColoring extends Disposable {
10811086
const length = srcData[srcOffset + 2];
10821087
const tokenTypeIndex = srcData[srcOffset + 3];
10831088
const tokenModifierSet = srcData[srcOffset + 4];
1084-
const metadata = styling.getMetadata(tokenTypeIndex, tokenModifierSet);
1089+
const metadata = styling.getMetadata(tokenTypeIndex, tokenModifierSet, languageId);
10851090

10861091
if (metadata !== Constants.NO_STYLING) {
10871092
if (areaLine === 0) {

‎src/vs/editor/standalone/browser/standaloneThemeServiceImpl.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -131,7 +131,7 @@ class StandaloneTheme implements IStandaloneTheme {
131131
return this._tokenTheme;
132132
}
133133

134-
public getTokenStyleMetadata(type: string, modifiers: string[]): ITokenStyle | undefined {
134+
public getTokenStyleMetadata(type: string, modifiers: string[], modelLanguage: string): ITokenStyle | undefined {
135135
return undefined;
136136
}
137137

‎src/vs/editor/standalone/test/browser/standaloneLanguages.test.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ suite('TokenizationSupport2Adapter', () => {
5656
throw new Error('Not implemented');
5757
},
5858

59-
getTokenStyleMetadata: (type: string, modifiers: string[]): ITokenStyle | undefined => {
59+
getTokenStyleMetadata: (type: string, modifiers: string[], modelLanguage: string): ITokenStyle | undefined => {
6060
return undefined;
6161
},
6262

‎src/vs/platform/theme/common/themeService.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,7 @@ export interface IColorTheme {
106106
/**
107107
* Returns the token style for a given classification. The result uses the <code>MetadataConsts</code> format
108108
*/
109-
getTokenStyleMetadata(type: string, modifiers: string[]): ITokenStyle | undefined;
109+
getTokenStyleMetadata(type: string, modifiers: string[], modelLanguage: string): ITokenStyle | undefined;
110110

111111
/**
112112
* List of all colors used with tokens. <code>getTokenStyleMetadata</code> references the colors by index into this list.

‎src/vs/platform/theme/common/tokenClassificationRegistry.ts‎

Lines changed: 65 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -13,15 +13,21 @@ import { Event, Emitter } from 'vs/base/common/event';
1313
import { IJSONSchema, IJSONSchemaMap } from 'vs/base/common/jsonSchema';
1414

1515
export const TOKEN_TYPE_WILDCARD = '*';
16+
export const TOKEN_CLASSIFIER_LANGUAGE_SEPARATOR = ':';
17+
export const CLASSIFIER_MODIFIER_SEPARATOR = '.';
1618

17-
// qualified string [type|*](.modifier)*
19+
// qualified string [type|*](.modifier)*(/language)!
1820
export type TokenClassificationString = string;
1921

20-
export const typeAndModifierIdPattern = '^\\w+[-_\\w+]*$';
22+
export const idPattern = '\\w+[-_\\w+]*';
23+
export const typeAndModifierIdPattern = `^${idPattern}$`;
24+
25+
export const selectorPattern = `^(${idPattern}|\\*)(\\${CLASSIFIER_MODIFIER_SEPARATOR}${idPattern})*(\\${TOKEN_CLASSIFIER_LANGUAGE_SEPARATOR}${idPattern})?$`;
26+
2127
export const fontStylePattern = '^(\\s*(-?italic|-?bold|-?underline))*\\s*$';
2228

2329
export interface TokenSelector {
24-
match(type: string, modifiers: string[]): number;
30+
match(type: string, modifiers: string[], language: string): number;
2531
readonly selectorString: string;
2632
}
2733

@@ -269,33 +275,39 @@ class TokenClassificationRegistry implements ITokenClassificationRegistry {
269275
}
270276

271277
public parseTokenSelector(selectorString: string): TokenSelector {
272-
const [selectorType, ...selectorModifiers] = selectorString.split('.');
278+
const selector = parseClassifierString(selectorString);
273279

274-
if (!selectorType) {
280+
if (!selector.type) {
275281
return {
276282
match: () => -1,
277283
selectorString
278284
};
279285
}
280286

281287
return {
282-
match: (type: string, modifiers: string[]) => {
288+
match: (type: string, modifiers: string[], language: string) => {
283289
let score = 0;
284-
if (selectorType !== TOKEN_TYPE_WILDCARD) {
290+
if (selector.language !== undefined) {
291+
if (selector.language !== language) {
292+
return -1;
293+
}
294+
score += 100;
295+
}
296+
if (selector.type !== TOKEN_TYPE_WILDCARD) {
285297
const hierarchy = this.getTypeHierarchy(type);
286-
const level = hierarchy.indexOf(selectorType);
298+
const level = hierarchy.indexOf(selector.type);
287299
if (level === -1) {
288300
return -1;
289301
}
290-
score = 100 - level;
302+
score += (100 - level);
291303
}
292304
// all selector modifiers must be present
293-
for (const selectorModifier of selectorModifiers) {
305+
for (const selectorModifier of selector.modifiers) {
294306
if (modifiers.indexOf(selectorModifier) === -1) {
295307
return -1;
296308
}
297309
}
298-
return score + selectorModifiers.length * 100;
310+
return score + selector.modifiers.length * 100;
299311
},
300312
selectorString
301313
};
@@ -366,15 +378,41 @@ class TokenClassificationRegistry implements ITokenClassificationRegistry {
366378

367379
}
368380

381+
const CHAR_LANGUAGE = TOKEN_CLASSIFIER_LANGUAGE_SEPARATOR.charCodeAt(0);
382+
const CHAR_MODIFIER = CLASSIFIER_MODIFIER_SEPARATOR.charCodeAt(0);
383+
384+
export function parseClassifierString(s: string): { type: string, modifiers: string[], language: string | undefined; } {
385+
let k = s.length;
386+
let language: string | undefined = undefined;
387+
const modifiers = [];
388+
389+
for (let i = k - 1; i >= 0; i--) {
390+
const ch = s.charCodeAt(i);
391+
if (ch === CHAR_LANGUAGE || ch === CHAR_MODIFIER) {
392+
const segment = s.substring(i + 1, k);
393+
k = i;
394+
if (ch === CHAR_LANGUAGE) {
395+
language = segment;
396+
} else {
397+
modifiers.push(segment);
398+
}
399+
}
400+
}
401+
const type = s.substring(0, k);
402+
return { type, modifiers, language };
403+
}
404+
369405

370-
const tokenClassificationRegistry = new TokenClassificationRegistry();
406+
let tokenClassificationRegistry = createDefaultTokenClassificationRegistry();
371407
platform.Registry.add(Extensions.TokenClassificationContribution, tokenClassificationRegistry);
372408

373-
registerDefaultClassifications();
374409

375-
function registerDefaultClassifications(): void {
410+
function createDefaultTokenClassificationRegistry(): TokenClassificationRegistry {
411+
412+
const registry = new TokenClassificationRegistry();
413+
376414
function registerTokenType(id: string, description: string, scopesToProbe: ProbeScope[] = [], superType?: string, deprecationMessage?: string): string {
377-
tokenClassificationRegistry.registerTokenType(id, description, superType, deprecationMessage);
415+
registry.registerTokenType(id, description, superType, deprecationMessage);
378416
if (scopesToProbe) {
379417
registerTokenStyleDefault(id, scopesToProbe);
380418
}
@@ -383,8 +421,8 @@ function registerDefaultClassifications(): void {
383421

384422
function registerTokenStyleDefault(selectorString: string, scopesToProbe: ProbeScope[]) {
385423
try {
386-
const selector = tokenClassificationRegistry.parseTokenSelector(selectorString);
387-
tokenClassificationRegistry.registerTokenStyleDefault(selector, { scopesToProbe });
424+
const selector = registry.parseTokenSelector(selectorString);
425+
registry.registerTokenStyleDefault(selector, { scopesToProbe });
388426
} catch (e) {
389427
console.log(e);
390428
}
@@ -422,18 +460,20 @@ function registerDefaultClassifications(): void {
422460

423461
// default token modifiers
424462

425-
tokenClassificationRegistry.registerTokenModifier('declaration', nls.localize('declaration', "Style for all symbol declarations."), undefined);
426-
tokenClassificationRegistry.registerTokenModifier('documentation', nls.localize('documentation', "Style to use for references in documentation."), undefined);
427-
tokenClassificationRegistry.registerTokenModifier('static', nls.localize('static', "Style to use for symbols that are static."), undefined);
428-
tokenClassificationRegistry.registerTokenModifier('abstract', nls.localize('abstract', "Style to use for symbols that are abstract."), undefined);
429-
tokenClassificationRegistry.registerTokenModifier('deprecated', nls.localize('deprecated', "Style to use for symbols that are deprecated."), undefined);
430-
tokenClassificationRegistry.registerTokenModifier('modification', nls.localize('modification', "Style to use for write accesses."), undefined);
431-
tokenClassificationRegistry.registerTokenModifier('async', nls.localize('async', "Style to use for symbols that are async."), undefined);
432-
tokenClassificationRegistry.registerTokenModifier('readonly', nls.localize('readonly', "Style to use for symbols that are readonly."), undefined);
463+
registry.registerTokenModifier('declaration', nls.localize('declaration', "Style for all symbol declarations."), undefined);
464+
registry.registerTokenModifier('documentation', nls.localize('documentation', "Style to use for references in documentation."), undefined);
465+
registry.registerTokenModifier('static', nls.localize('static', "Style to use for symbols that are static."), undefined);
466+
registry.registerTokenModifier('abstract', nls.localize('abstract', "Style to use for symbols that are abstract."), undefined);
467+
registry.registerTokenModifier('deprecated', nls.localize('deprecated', "Style to use for symbols that are deprecated."), undefined);
468+
registry.registerTokenModifier('modification', nls.localize('modification', "Style to use for write accesses."), undefined);
469+
registry.registerTokenModifier('async', nls.localize('async', "Style to use for symbols that are async."), undefined);
470+
registry.registerTokenModifier('readonly', nls.localize('readonly', "Style to use for symbols that are readonly."), undefined);
433471

434472

435473
registerTokenStyleDefault('variable.readonly', [['variable.other.constant']]);
436474
registerTokenStyleDefault('property.readonly', [['variable.other.constant.property']]);
475+
476+
return registry;
437477
}
438478

439479
export function getTokenClassificationRegistry(): ITokenClassificationRegistry {

‎src/vs/platform/theme/test/common/testThemeService.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ export class TestColorTheme implements IColorTheme {
2424
throw new Error('Method not implemented.');
2525
}
2626

27-
getTokenStyleMetadata(type: string, modifiers: string[]): ITokenStyle | undefined {
27+
getTokenStyleMetadata(type: string, modifiers: string[], modelLanguage: string): ITokenStyle | undefined {
2828
return undefined;
2929
}
3030

‎src/vs/workbench/contrib/codeEditor/browser/inspectEditorTokens/inspectEditorTokens.ts‎

Lines changed: 4 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -497,6 +497,7 @@ class InspectEditorTokensWidget extends Disposable implements IContentWidget {
497497

498498
private _getSemanticTokenAtPosition(semanticTokens: SemanticTokensResult, pos: Position): ISemanticTokenInfo | null {
499499
const tokenData = semanticTokens.tokens.data;
500+
const defaultLanguage = this._model.getLanguageIdentifier().language;
500501
let lastLine = 0;
501502
let lastCharacter = 0;
502503
const posLine = pos.lineNumber - 1, posCharacter = pos.column - 1; // to 0-based position
@@ -511,7 +512,7 @@ class InspectEditorTokensWidget extends Disposable implements IContentWidget {
511512
const definitions = {};
512513
const colorMap = this._themeService.getColorTheme().tokenColorMap;
513514
const theme = this._themeService.getColorTheme() as ColorThemeData;
514-
const tokenStyle = theme.getTokenStyleMetadata(type, modifiers, true, definitions);
515+
const tokenStyle = theme.getTokenStyleMetadata(type, modifiers, defaultLanguage, true, definitions);
515516

516517
let metadata: IDecodedMetadata | undefined = undefined;
517518
if (tokenStyle) {
@@ -556,16 +557,9 @@ class InspectEditorTokensWidget extends Disposable implements IContentWidget {
556557
return `Color theme: ${definition.selector.selectorString} - ${this._renderStyleProperty(definition.style, property)}`;
557558
}
558559
return '';
559-
} else if (typeof definition === 'string') {
560-
const [type, ...modifiers] = definition.split('.');
561-
const definitions: TokenStyleDefinitions = {};
562-
const m = theme.getTokenStyleMetadata(type, modifiers, true, definitions);
563-
if (m && definitions.foreground) {
564-
return this._renderTokenStyleDefinition(definitions[property], property);
565-
}
566-
return '';
567560
} else {
568-
return this._renderStyleProperty(definition, property);
561+
const style = theme.resolveTokenStyleValue(definition);
562+
return `Default: ${style ? this._renderStyleProperty(style, property) : ''}`;
569563
}
570564
}
571565

0 commit comments

Comments
 (0)