Skip to content

Commit 0cf78e1

Browse files
aeschliaeschli
committed
language in token
1 parent b089ff1 commit 0cf78e1

10 files changed

Lines changed: 171 additions & 82 deletions

File tree

‎src/vs/editor/common/services/modelServiceImpl.ts‎

Lines changed: 19 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -550,12 +550,14 @@ const enum Constants {
550550
class HashTableEntry {
551551
public readonly tokenTypeIndex: number;
552552
public readonly tokenModifierSet: number;
553+
public readonly languageId: number;
553554
public readonly metadata: number;
554555
public next: HashTableEntry | null;
555556

556-
constructor(tokenTypeIndex: number, tokenModifierSet: number, metadata: number) {
557+
constructor(tokenTypeIndex: number, tokenModifierSet: number, languageId: number, metadata: number) {
557558
this.tokenTypeIndex = tokenTypeIndex;
558559
this.tokenModifierSet = tokenModifierSet;
560+
this.languageId = languageId;
559561
this.metadata = metadata;
560562
this.next = null;
561563
}
@@ -586,16 +588,17 @@ class HashTable {
586588
}
587589
}
588590

589-
private _hashFunc(tokenTypeIndex: number, tokenModifierSet: number): number {
590-
return ((((tokenTypeIndex << 5) - tokenTypeIndex) + tokenModifierSet) | 0) % this._currentLength; // tokenTypeIndex * 31 + tokenModifierSet, keep as int32
591+
private _hashFunc(tokenTypeIndex: number, tokenModifierSet: number, languageId: number): number {
592+
const hash = (n1: number, n2: number) => (((n1 << 5) - n1) + n2) | 0; // n1 * 31 + n2, keep as int32
593+
return hash(hash(tokenTypeIndex, tokenModifierSet), languageId) % this._currentLength;
591594
}
592595

593-
public get(tokenTypeIndex: number, tokenModifierSet: number): HashTableEntry | null {
594-
const hash = this._hashFunc(tokenTypeIndex, tokenModifierSet);
596+
public get(tokenTypeIndex: number, tokenModifierSet: number, languageId: number): HashTableEntry | null {
597+
const hash = this._hashFunc(tokenTypeIndex, tokenModifierSet, languageId);
595598

596599
let p = this._elements[hash];
597600
while (p) {
598-
if (p.tokenTypeIndex === tokenTypeIndex && p.tokenModifierSet === tokenModifierSet) {
601+
if (p.tokenTypeIndex === tokenTypeIndex && p.tokenModifierSet === tokenModifierSet && p.languageId === languageId) {
599602
return p;
600603
}
601604
p = p.next;
@@ -604,7 +607,7 @@ class HashTable {
604607
return null;
605608
}
606609

607-
public add(tokenTypeIndex: number, tokenModifierSet: number, metadata: number): void {
610+
public add(tokenTypeIndex: number, tokenModifierSet: number, languageId: number, metadata: number): void {
608611
this._elementsCount++;
609612
if (this._growCount !== 0 && this._elementsCount >= this._growCount) {
610613
// expand!
@@ -626,11 +629,11 @@ class HashTable {
626629
}
627630
}
628631
}
629-
this._add(new HashTableEntry(tokenTypeIndex, tokenModifierSet, metadata));
632+
this._add(new HashTableEntry(tokenTypeIndex, tokenModifierSet, languageId, metadata));
630633
}
631634

632635
private _add(element: HashTableEntry): void {
633-
const hash = this._hashFunc(element.tokenTypeIndex, element.tokenModifierSet);
636+
const hash = this._hashFunc(element.tokenTypeIndex, element.tokenModifierSet, element.languageId);
634637
element.next = this._elements[hash];
635638
this._elements[hash] = element;
636639
}
@@ -648,8 +651,8 @@ class SemanticColoringProviderStyling {
648651
this._hashTable = new HashTable();
649652
}
650653

651-
public getMetadata(tokenTypeIndex: number, tokenModifierSet: number): number {
652-
const entry = this._hashTable.get(tokenTypeIndex, tokenModifierSet);
654+
public getMetadata(tokenTypeIndex: number, tokenModifierSet: number, languageId: LanguageIdentifier): number {
655+
const entry = this._hashTable.get(tokenTypeIndex, tokenModifierSet, languageId.id);
653656
let metadata: number;
654657
if (entry) {
655658
metadata = entry.metadata;
@@ -664,7 +667,7 @@ class SemanticColoringProviderStyling {
664667
modifierSet = modifierSet >> 1;
665668
}
666669

667-
const tokenStyle = this._themeService.getColorTheme().getTokenStyleMetadata(tokenType, tokenModifiers);
670+
const tokenStyle = this._themeService.getColorTheme().getTokenStyleMetadata(tokenType, tokenModifiers, languageId.language);
668671
if (typeof tokenStyle === 'undefined') {
669672
metadata = Constants.NO_STYLING;
670673
} else {
@@ -690,7 +693,7 @@ class SemanticColoringProviderStyling {
690693
metadata = Constants.NO_STYLING;
691694
}
692695
}
693-
this._hashTable.add(tokenTypeIndex, tokenModifierSet, metadata);
696+
this._hashTable.add(tokenTypeIndex, tokenModifierSet, languageId.id, metadata);
694697
}
695698
if (this._logService.getLevel() === LogLevel.Trace) {
696699
const type = this._legend.tokenTypes[tokenTypeIndex];
@@ -931,6 +934,8 @@ class ModelSemanticColoring extends Disposable {
931934

932935
const result: MultilineTokens2[] = [];
933936

937+
const languageId = this._model.getLanguageIdentifier();
938+
934939
let tokenIndex = 0;
935940
let lastLineNumber = 1;
936941
let lastStartCharacter = 0;
@@ -970,7 +975,7 @@ class ModelSemanticColoring extends Disposable {
970975
const length = srcData[srcOffset + 2];
971976
const tokenTypeIndex = srcData[srcOffset + 3];
972977
const tokenModifierSet = srcData[srcOffset + 4];
973-
const metadata = styling.getMetadata(tokenTypeIndex, tokenModifierSet);
978+
const metadata = styling.getMetadata(tokenTypeIndex, tokenModifierSet, languageId);
974979

975980
if (metadata !== Constants.NO_STYLING) {
976981
if (areaLine === 0) {

‎src/vs/editor/standalone/browser/standaloneThemeServiceImpl.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -131,7 +131,7 @@ class StandaloneTheme implements IStandaloneTheme {
131131
return this._tokenTheme;
132132
}
133133

134-
public getTokenStyleMetadata(type: string, modifiers: string[]): ITokenStyle | undefined {
134+
public getTokenStyleMetadata(type: string, modifiers: string[], modelLanguage: string): ITokenStyle | undefined {
135135
return undefined;
136136
}
137137

‎src/vs/editor/standalone/test/browser/standaloneLanguages.test.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ suite('TokenizationSupport2Adapter', () => {
5656
throw new Error('Not implemented');
5757
},
5858

59-
getTokenStyleMetadata: (type: string, modifiers: string[]): ITokenStyle | undefined => {
59+
getTokenStyleMetadata: (type: string, modifiers: string[], modelLanguage: string): ITokenStyle | undefined => {
6060
return undefined;
6161
},
6262

‎src/vs/platform/theme/common/themeService.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,7 @@ export interface IColorTheme {
106106
/**
107107
* Returns the token style for a given classification. The result uses the <code>MetadataConsts</code> format
108108
*/
109-
getTokenStyleMetadata(type: string, modifiers: string[]): ITokenStyle | undefined;
109+
getTokenStyleMetadata(type: string, modifiers: string[], modelLanguage: string): ITokenStyle | undefined;
110110

111111
/**
112112
* List of all colors used with tokens. <code>getTokenStyleMetadata</code> references the colors by index into this list.

‎src/vs/platform/theme/common/tokenClassificationRegistry.ts‎

Lines changed: 65 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -13,15 +13,21 @@ import { Event, Emitter } from 'vs/base/common/event';
1313
import { IJSONSchema, IJSONSchemaMap } from 'vs/base/common/jsonSchema';
1414

1515
export const TOKEN_TYPE_WILDCARD = '*';
16+
export const TOKEN_CLASSIFIER_LANGUAGE_SEPARATOR = ':';
17+
export const CLASSIFIER_MODIFIER_SEPARATOR = '.';
1618

17-
// qualified string [type|*](.modifier)*
19+
// qualified string [type|*](.modifier)*(/language)!
1820
export type TokenClassificationString = string;
1921

20-
export const typeAndModifierIdPattern = '^\\w+[-_\\w+]*$';
22+
export const idPattern = '\\w+[-_\\w+]*';
23+
export const typeAndModifierIdPattern = `^${idPattern}$`;
24+
25+
export const selectorPattern = `^(${idPattern}|\\*)(\\${CLASSIFIER_MODIFIER_SEPARATOR}${idPattern})*(\\${TOKEN_CLASSIFIER_LANGUAGE_SEPARATOR}${idPattern})?$`;
26+
2127
export const fontStylePattern = '^(\\s*(-?italic|-?bold|-?underline))*\\s*$';
2228

2329
export interface TokenSelector {
24-
match(type: string, modifiers: string[]): number;
30+
match(type: string, modifiers: string[], language: string): number;
2531
readonly selectorString: string;
2632
}
2733

@@ -269,33 +275,39 @@ class TokenClassificationRegistry implements ITokenClassificationRegistry {
269275
}
270276

271277
public parseTokenSelector(selectorString: string): TokenSelector {
272-
const [selectorType, ...selectorModifiers] = selectorString.split('.');
278+
const selector = parseClassifierString(selectorString);
273279

274-
if (!selectorType) {
280+
if (!selector.type) {
275281
return {
276282
match: () => -1,
277283
selectorString
278284
};
279285
}
280286

281287
return {
282-
match: (type: string, modifiers: string[]) => {
288+
match: (type: string, modifiers: string[], language: string) => {
283289
let score = 0;
284-
if (selectorType !== TOKEN_TYPE_WILDCARD) {
290+
if (selector.language !== undefined) {
291+
if (selector.language !== language) {
292+
return -1;
293+
}
294+
score += 100;
295+
}
296+
if (selector.type !== TOKEN_TYPE_WILDCARD) {
285297
const hierarchy = this.getTypeHierarchy(type);
286-
const level = hierarchy.indexOf(selectorType);
298+
const level = hierarchy.indexOf(selector.type);
287299
if (level === -1) {
288300
return -1;
289301
}
290-
score = 100 - level;
302+
score += (100 - level);
291303
}
292304
// all selector modifiers must be present
293-
for (const selectorModifier of selectorModifiers) {
305+
for (const selectorModifier of selector.modifiers) {
294306
if (modifiers.indexOf(selectorModifier) === -1) {
295307
return -1;
296308
}
297309
}
298-
return score + selectorModifiers.length * 100;
310+
return score + selector.modifiers.length * 100;
299311
},
300312
selectorString
301313
};
@@ -366,15 +378,41 @@ class TokenClassificationRegistry implements ITokenClassificationRegistry {
366378

367379
}
368380

381+
const CHAR_LANGUAGE = TOKEN_CLASSIFIER_LANGUAGE_SEPARATOR.charCodeAt(0);
382+
const CHAR_MODIFIER = CLASSIFIER_MODIFIER_SEPARATOR.charCodeAt(0);
383+
384+
export function parseClassifierString(s: string): { type: string, modifiers: string[], language: string | undefined; } {
385+
let k = s.length;
386+
let language: string | undefined = undefined;
387+
const modifiers = [];
388+
389+
for (let i = k - 1; i >= 0; i--) {
390+
const ch = s.charCodeAt(i);
391+
if (ch === CHAR_LANGUAGE || ch === CHAR_MODIFIER) {
392+
const segment = s.substring(i + 1, k);
393+
k = i;
394+
if (ch === CHAR_LANGUAGE) {
395+
language = segment;
396+
} else {
397+
modifiers.push(segment);
398+
}
399+
}
400+
}
401+
const type = s.substring(0, k);
402+
return { type, modifiers, language };
403+
}
404+
369405

370-
const tokenClassificationRegistry = new TokenClassificationRegistry();
406+
let tokenClassificationRegistry = createDefaultTokenClassificationRegistry();
371407
platform.Registry.add(Extensions.TokenClassificationContribution, tokenClassificationRegistry);
372408

373-
registerDefaultClassifications();
374409

375-
function registerDefaultClassifications(): void {
410+
function createDefaultTokenClassificationRegistry(): TokenClassificationRegistry {
411+
412+
const registry = new TokenClassificationRegistry();
413+
376414
function registerTokenType(id: string, description: string, scopesToProbe: ProbeScope[] = [], superType?: string, deprecationMessage?: string): string {
377-
tokenClassificationRegistry.registerTokenType(id, description, superType, deprecationMessage);
415+
registry.registerTokenType(id, description, superType, deprecationMessage);
378416
if (scopesToProbe) {
379417
registerTokenStyleDefault(id, scopesToProbe);
380418
}
@@ -383,8 +421,8 @@ function registerDefaultClassifications(): void {
383421

384422
function registerTokenStyleDefault(selectorString: string, scopesToProbe: ProbeScope[]) {
385423
try {
386-
const selector = tokenClassificationRegistry.parseTokenSelector(selectorString);
387-
tokenClassificationRegistry.registerTokenStyleDefault(selector, { scopesToProbe });
424+
const selector = registry.parseTokenSelector(selectorString);
425+
registry.registerTokenStyleDefault(selector, { scopesToProbe });
388426
} catch (e) {
389427
console.log(e);
390428
}
@@ -422,18 +460,20 @@ function registerDefaultClassifications(): void {
422460

423461
// default token modifiers
424462

425-
tokenClassificationRegistry.registerTokenModifier('declaration', nls.localize('declaration', "Style for all symbol declarations."), undefined);
426-
tokenClassificationRegistry.registerTokenModifier('documentation', nls.localize('documentation', "Style to use for references in documentation."), undefined);
427-
tokenClassificationRegistry.registerTokenModifier('static', nls.localize('static', "Style to use for symbols that are static."), undefined);
428-
tokenClassificationRegistry.registerTokenModifier('abstract', nls.localize('abstract', "Style to use for symbols that are abstract."), undefined);
429-
tokenClassificationRegistry.registerTokenModifier('deprecated', nls.localize('deprecated', "Style to use for symbols that are deprecated."), undefined);
430-
tokenClassificationRegistry.registerTokenModifier('modification', nls.localize('modification', "Style to use for write accesses."), undefined);
431-
tokenClassificationRegistry.registerTokenModifier('async', nls.localize('async', "Style to use for symbols that are async."), undefined);
432-
tokenClassificationRegistry.registerTokenModifier('readonly', nls.localize('readonly', "Style to use for symbols that are readonly."), undefined);
463+
registry.registerTokenModifier('declaration', nls.localize('declaration', "Style for all symbol declarations."), undefined);
464+
registry.registerTokenModifier('documentation', nls.localize('documentation', "Style to use for references in documentation."), undefined);
465+
registry.registerTokenModifier('static', nls.localize('static', "Style to use for symbols that are static."), undefined);
466+
registry.registerTokenModifier('abstract', nls.localize('abstract', "Style to use for symbols that are abstract."), undefined);
467+
registry.registerTokenModifier('deprecated', nls.localize('deprecated', "Style to use for symbols that are deprecated."), undefined);
468+
registry.registerTokenModifier('modification', nls.localize('modification', "Style to use for write accesses."), undefined);
469+
registry.registerTokenModifier('async', nls.localize('async', "Style to use for symbols that are async."), undefined);
470+
registry.registerTokenModifier('readonly', nls.localize('readonly', "Style to use for symbols that are readonly."), undefined);
433471

434472

435473
registerTokenStyleDefault('variable.readonly', [['variable.other.constant']]);
436474
registerTokenStyleDefault('property.readonly', [['variable.other.constant.property']]);
475+
476+
return registry;
437477
}
438478

439479
export function getTokenClassificationRegistry(): ITokenClassificationRegistry {

‎src/vs/platform/theme/test/common/testThemeService.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ export class TestColorTheme implements IColorTheme {
2424
throw new Error('Method not implemented.');
2525
}
2626

27-
getTokenStyleMetadata(type: string, modifiers: string[]): ITokenStyle | undefined {
27+
getTokenStyleMetadata(type: string, modifiers: string[], modelLanguage: string): ITokenStyle | undefined {
2828
return undefined;
2929
}
3030

‎src/vs/workbench/contrib/codeEditor/browser/inspectEditorTokens/inspectEditorTokens.ts‎

Lines changed: 4 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -491,6 +491,7 @@ class InspectEditorTokensWidget extends Disposable implements IContentWidget {
491491

492492
private _getSemanticTokenAtPosition(semanticTokens: SemanticTokensResult, pos: Position): ISemanticTokenInfo | null {
493493
const tokenData = semanticTokens.tokens.data;
494+
const defaultLanguage = this._model.getLanguageIdentifier().language;
494495
let lastLine = 0;
495496
let lastCharacter = 0;
496497
const posLine = pos.lineNumber - 1, posCharacter = pos.column - 1; // to 0-based position
@@ -505,7 +506,7 @@ class InspectEditorTokensWidget extends Disposable implements IContentWidget {
505506
const definitions = {};
506507
const colorMap = this._themeService.getColorTheme().tokenColorMap;
507508
const theme = this._themeService.getColorTheme() as ColorThemeData;
508-
const tokenStyle = theme.getTokenStyleMetadata(type, modifiers, true, definitions);
509+
const tokenStyle = theme.getTokenStyleMetadata(type, modifiers, defaultLanguage, true, definitions);
509510

510511
let metadata: IDecodedMetadata | undefined = undefined;
511512
if (tokenStyle) {
@@ -550,16 +551,9 @@ class InspectEditorTokensWidget extends Disposable implements IContentWidget {
550551
return `Color theme: ${definition.selector.selectorString} - ${this._renderStyleProperty(definition.style, property)}`;
551552
}
552553
return '';
553-
} else if (typeof definition === 'string') {
554-
const [type, ...modifiers] = definition.split('.');
555-
const definitions: TokenStyleDefinitions = {};
556-
const m = theme.getTokenStyleMetadata(type, modifiers, true, definitions);
557-
if (m && definitions.foreground) {
558-
return this._renderTokenStyleDefinition(definitions[property], property);
559-
}
560-
return '';
561554
} else {
562-
return this._renderStyleProperty(definition, property);
555+
const style = theme.resolveTokenStyleValue(definition);
556+
return `Default: ${style ? this._renderStyleProperty(style, property) : ''}`;
563557
}
564558
}
565559

0 commit comments

Comments
 (0)