Skip to content

Commit fe5024c

Browse files
aeschliaeschli
committed
Handle invalid token type/modifier indexes. Fixes microsoft#96540 
1 parent ddff80b commit fe5024c

4 files changed

Lines changed: 77 additions & 45 deletions

File tree

‎extensions/vscode-colorize-tests/src/colorizerTestMain.ts‎

Lines changed: 14 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -22,24 +22,31 @@ export function activate(context: vscode.ExtensionContext): any {
2222
function addToken(value: string, startLine: number, startCharacter: number, length: number) {
2323
const [type, ...modifiers] = value.split('.');
2424

25+
const selectedModifiers = [];
26+
2527
let tokenType = legend.tokenTypes.indexOf(type);
2628
if (tokenType === -1) {
27-
return;
29+
if (type === 'notInLegend') {
30+
tokenType = tokenTypes.length + 2;
31+
} else {
32+
return;
33+
}
2834
}
2935

3036
let tokenModifiers = 0;
31-
for (let i = 0; i < modifiers.length; i++) {
32-
const index = legend.tokenModifiers.indexOf(modifiers[i]);
37+
for (const modifier of modifiers) {
38+
const index = legend.tokenModifiers.indexOf(modifier);
3339
if (index !== -1) {
3440
tokenModifiers = tokenModifiers | 1 << index;
41+
selectedModifiers.push(modifier);
42+
} else if (modifier === 'notInLegend') {
43+
tokenModifiers = tokenModifiers | 1 << (legend.tokenModifiers.length + 2);
44+
selectedModifiers.push(modifier);
3545
}
3646
}
37-
38-
3947
builder.push(startLine, startCharacter, length, tokenType, tokenModifiers);
4048

41-
const selectedModifiers = legend.tokenModifiers.filter((_val, bit) => tokenModifiers & (1 << bit)).join(' ');
42-
outputChannel.appendLine(`line: ${startLine}, character: ${startCharacter}, length ${length}, ${legend.tokenTypes[tokenType]} (${tokenType}), ${selectedModifiers} ${tokenModifiers.toString(2)}`);
49+
outputChannel.appendLine(`line: ${startLine}, character: ${startCharacter}, length ${length}, ${type} (${tokenType}), ${selectedModifiers} ${tokenModifiers.toString(2)}`);
4350
}
4451

4552
outputChannel.appendLine('---');

‎extensions/vscode-colorize-tests/test/semantic-test/semantic-test.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,6 @@
44
"variable.declaration", "parameterNames",
55
"function.member.declaration",
66
"interface.declaration",
7-
"function.member.declaration", "testToken.testModifier"
7+
"function.member.declaration", "function.notInLegend"
88

99
]

‎src/vs/editor/common/services/semanticTokensProviderStyling.ts‎

Lines changed: 50 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -29,50 +29,65 @@ export class SemanticTokensProviderStyling {
2929
let metadata: number;
3030
if (entry) {
3131
metadata = entry.metadata;
32+
if (this._logService.getLevel() === LogLevel.Trace) {
33+
this._logService.trace(`SemanticTokensProviderStyling [CACHED] ${tokenTypeIndex} / ${tokenModifierSet}: foreground ${TokenMetadata.getForeground(metadata)}, fontStyle ${TokenMetadata.getFontStyle(metadata).toString(2)}`);
34+
}
3235
} else {
33-
const tokenType = this._legend.tokenTypes[tokenTypeIndex];
36+
let tokenType = this._legend.tokenTypes[tokenTypeIndex];
3437
const tokenModifiers: string[] = [];
35-
let modifierSet = tokenModifierSet;
36-
for (let modifierIndex = 0; modifierSet > 0 && modifierIndex < this._legend.tokenModifiers.length; modifierIndex++) {
37-
if (modifierSet & 1) {
38-
tokenModifiers.push(this._legend.tokenModifiers[modifierIndex]);
39-
}
40-
modifierSet = modifierSet >> 1;
41-
}
42-
43-
const tokenStyle = this._themeService.getColorTheme().getTokenStyleMetadata(tokenType, tokenModifiers, languageId.language);
44-
if (typeof tokenStyle === 'undefined') {
45-
metadata = SemanticTokensProviderStylingConstants.NO_STYLING;
46-
} else {
47-
metadata = 0;
48-
if (typeof tokenStyle.italic !== 'undefined') {
49-
const italicBit = (tokenStyle.italic ? FontStyle.Italic : 0) << MetadataConsts.FONT_STYLE_OFFSET;
50-
metadata |= italicBit | MetadataConsts.SEMANTIC_USE_ITALIC;
51-
}
52-
if (typeof tokenStyle.bold !== 'undefined') {
53-
const boldBit = (tokenStyle.bold ? FontStyle.Bold : 0) << MetadataConsts.FONT_STYLE_OFFSET;
54-
metadata |= boldBit | MetadataConsts.SEMANTIC_USE_BOLD;
55-
}
56-
if (typeof tokenStyle.underline !== 'undefined') {
57-
const underlineBit = (tokenStyle.underline ? FontStyle.Underline : 0) << MetadataConsts.FONT_STYLE_OFFSET;
58-
metadata |= underlineBit | MetadataConsts.SEMANTIC_USE_UNDERLINE;
38+
if (tokenType) {
39+
let modifierSet = tokenModifierSet;
40+
for (let modifierIndex = 0; modifierSet > 0 && modifierIndex < this._legend.tokenModifiers.length; modifierIndex++) {
41+
if (modifierSet & 1) {
42+
tokenModifiers.push(this._legend.tokenModifiers[modifierIndex]);
43+
}
44+
modifierSet = modifierSet >> 1;
5945
}
60-
if (tokenStyle.foreground) {
61-
const foregroundBits = (tokenStyle.foreground) << MetadataConsts.FOREGROUND_OFFSET;
62-
metadata |= foregroundBits | MetadataConsts.SEMANTIC_USE_FOREGROUND;
46+
if (modifierSet > 0 && this._logService.getLevel() === LogLevel.Trace) {
47+
this._logService.trace(`SemanticTokensProviderStyling: unknown token modifier index: ${tokenModifierSet.toString(2)} for legend: ${JSON.stringify(this._legend.tokenModifiers)}`);
48+
tokenModifiers.push('not-in-legend');
6349
}
64-
if (metadata === 0) {
65-
// Nothing!
50+
51+
const tokenStyle = this._themeService.getColorTheme().getTokenStyleMetadata(tokenType, tokenModifiers, languageId.language);
52+
if (typeof tokenStyle === 'undefined') {
6653
metadata = SemanticTokensProviderStylingConstants.NO_STYLING;
54+
} else {
55+
metadata = 0;
56+
if (typeof tokenStyle.italic !== 'undefined') {
57+
const italicBit = (tokenStyle.italic ? FontStyle.Italic : 0) << MetadataConsts.FONT_STYLE_OFFSET;
58+
metadata |= italicBit | MetadataConsts.SEMANTIC_USE_ITALIC;
59+
}
60+
if (typeof tokenStyle.bold !== 'undefined') {
61+
const boldBit = (tokenStyle.bold ? FontStyle.Bold : 0) << MetadataConsts.FONT_STYLE_OFFSET;
62+
metadata |= boldBit | MetadataConsts.SEMANTIC_USE_BOLD;
63+
}
64+
if (typeof tokenStyle.underline !== 'undefined') {
65+
const underlineBit = (tokenStyle.underline ? FontStyle.Underline : 0) << MetadataConsts.FONT_STYLE_OFFSET;
66+
metadata |= underlineBit | MetadataConsts.SEMANTIC_USE_UNDERLINE;
67+
}
68+
if (tokenStyle.foreground) {
69+
const foregroundBits = (tokenStyle.foreground) << MetadataConsts.FOREGROUND_OFFSET;
70+
metadata |= foregroundBits | MetadataConsts.SEMANTIC_USE_FOREGROUND;
71+
}
72+
if (metadata === 0) {
73+
// Nothing!
74+
metadata = SemanticTokensProviderStylingConstants.NO_STYLING;
75+
}
76+
}
77+
} else {
78+
if (this._logService.getLevel() === LogLevel.Trace) {
79+
this._logService.trace(`SemanticTokensProviderStyling: unknown token type index: ${tokenTypeIndex} for legend: ${JSON.stringify(this._legend.tokenTypes)}`);
6780
}
81+
metadata = SemanticTokensProviderStylingConstants.NO_STYLING;
82+
tokenType = 'not-in-legend';
6883
}
6984
this._hashTable.add(tokenTypeIndex, tokenModifierSet, languageId.id, metadata);
85+
86+
if (this._logService.getLevel() === LogLevel.Trace) {
87+
this._logService.trace(`SemanticTokensProviderStyling ${tokenTypeIndex} (${tokenType}) / ${tokenModifierSet} (${tokenModifiers.join(' ')}): foreground ${TokenMetadata.getForeground(metadata)}, fontStyle ${TokenMetadata.getFontStyle(metadata).toString(2)}`);
88+
}
7089
}
71-
if (this._logService.getLevel() === LogLevel.Trace) {
72-
const type = this._legend.tokenTypes[tokenTypeIndex];
73-
const modifiers = tokenModifierSet ? ' ' + this._legend.tokenModifiers.filter((_, i) => tokenModifierSet & (1 << i)).join(' ') : '';
74-
this._logService.trace(`tokenStyleMetadata ${entry ? '[CACHED] ' : ''}${type}${modifiers}: foreground ${TokenMetadata.getForeground(metadata)}, fontStyle ${TokenMetadata.getFontStyle(metadata).toString(2)}`);
75-
}
90+
7691
return metadata;
7792
}
7893
}

‎src/vs/workbench/contrib/codeEditor/browser/inspectEditorTokens/inspectEditorTokens.ts‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -514,8 +514,18 @@ class InspectEditorTokensWidget extends Disposable implements IContentWidget {
514514
const line = lastLine + lineDelta; // 0-based
515515
const character = lineDelta === 0 ? lastCharacter + charDelta : charDelta; // 0-based
516516
if (posLine === line && character <= posCharacter && posCharacter < character + len) {
517-
const type = semanticTokens.legend.tokenTypes[typeIdx];
518-
const modifiers = semanticTokens.legend.tokenModifiers.filter((_, k) => modSet & 1 << k);
517+
const type = semanticTokens.legend.tokenTypes[typeIdx] || 'not in legend (ignored)';
518+
const modifiers = [];
519+
let modifierSet = modSet;
520+
for (let modifierIndex = 0; modifierSet > 0 && modifierIndex < semanticTokens.legend.tokenModifiers.length; modifierIndex++) {
521+
if (modifierSet & 1) {
522+
modifiers.push(semanticTokens.legend.tokenModifiers[modifierIndex]);
523+
}
524+
modifierSet = modifierSet >> 1;
525+
}
526+
if (modifierSet > 0) {
527+
modifiers.push('not in legend (ignored)');
528+
}
519529
const range = new Range(line + 1, character + 1, line + 1, character + 1 + len);
520530
const definitions = {};
521531
const colorMap = this._themeService.getColorTheme().tokenColorMap;

0 commit comments

Comments
 (0)