Skip to content

Commit f24b58c

Browse files
aeschliaeschli
committed
ts sem tokens in html
1 parent df41996 commit f24b58c

10 files changed

Lines changed: 253 additions & 116 deletions

File tree

‎.vscode/launch.json‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -233,6 +233,21 @@
233233
"order": 10
234234
}
235235
},
236+
{
237+
"type": "node",
238+
"request": "launch",
239+
"name": "HTML Unit Tests",
240+
"program": "${workspaceFolder}/extensions/html-language-features/server/node_modules/mocha/bin/_mocha",
241+
"stopOnEntry": false,
242+
"cwd": "${workspaceFolder}/extensions/html-language-features/server",
243+
"outFiles": [
244+
"${workspaceFolder}/extensions/html-language-features/server/out/**/*.js"
245+
],
246+
"presentation": {
247+
"group": "5_tests",
248+
"order": 10
249+
}
250+
},
236251
{
237252
"type": "extensionHost",
238253
"request": "launch",

‎extensions/html-language-features/server/src/htmlServerMain.ts‎

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ import { formatError, runSafe, runSafeAsync } from './utils/runner';
2323
import { getFoldingRanges } from './modes/htmlFolding';
2424
import { getDataProviders } from './customData';
2525
import { getSelectionRanges } from './modes/selectionRanges';
26+
import { SemanticTokenProvider, newSemanticTokenProvider } from './modes/semanticTokens';
2627

2728
namespace TagCloseRequest {
2829
export const type: RequestType<TextDocumentPositionParams, string | null, any, any> = new RequestType('html/tag');
@@ -530,26 +531,27 @@ connection.onRequest(MatchingTagPositionRequest.type, (params, token) => {
530531
}, null, `Error while computing matching tag position for ${params.textDocument.uri}`, token);
531532
});
532533

534+
let semanticTokensProvider: SemanticTokenProvider | undefined;
535+
function getSemanticTokenProvider() {
536+
if (!semanticTokensProvider) {
537+
semanticTokensProvider = newSemanticTokenProvider(languageModes);
538+
}
539+
return semanticTokensProvider;
540+
}
541+
533542
connection.onRequest(SemanticTokenRequest.type, (params, token) => {
534543
return runSafe(() => {
535544
const document = documents.get(params.textDocument.uri);
536545
if (document) {
537-
const jsMode = languageModes.getMode('javascript');
538-
if (jsMode && jsMode.getSemanticTokens) {
539-
return jsMode.getSemanticTokens(document, params.ranges);
540-
}
546+
return getSemanticTokenProvider().getSemanticTokens(document, params.ranges);
541547
}
542548
return null;
543549
}, null, `Error while computing semantic tokens for ${params.textDocument.uri}`, token);
544550
});
545551

546552
connection.onRequest(SemanticTokenLegendRequest.type, (_params, token) => {
547553
return runSafe(() => {
548-
const jsMode = languageModes.getMode('javascript');
549-
if (jsMode && jsMode.getSemanticTokenLegend) {
550-
return jsMode.getSemanticTokenLegend();
551-
}
552-
return null;
554+
return getSemanticTokenProvider().legend;
553555
}, null, `Error while computing semantic tokens legend`, token);
554556
});
555557

‎extensions/html-language-features/server/src/modes/embeddedSupport.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,8 @@ export function getDocumentRegions(languageService: LanguageService, document: T
5858
} else if (lastAttributeName === 'type' && lastTagName.toLowerCase() === 'script') {
5959
if (/["'](module|(text|application)\/(java|ecma)script|text\/babel)["']/.test(scanner.getTokenText())) {
6060
languageIdFromType = 'javascript';
61+
} else if (/["']text\/typescript["']/.test(scanner.getTokenText())) {
62+
languageIdFromType = 'typescript';
6163
} else {
6264
languageIdFromType = undefined;
6365
}

‎extensions/html-language-features/server/src/modes/javascriptMode.ts‎

Lines changed: 26 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ import {
88
SymbolInformation, SymbolKind, CompletionItem, Location, SignatureHelp, SignatureInformation, ParameterInformation,
99
Definition, TextEdit, TextDocument, Diagnostic, DiagnosticSeverity, Range, CompletionItemKind, Hover, MarkedString,
1010
DocumentHighlight, DocumentHighlightKind, CompletionList, Position, FormattingOptions, FoldingRange, FoldingRangeKind, SelectionRange,
11-
LanguageMode, Settings
11+
LanguageMode, Settings, SemanticTokenData
1212
} from './languageModes';
1313
import { getWordAtText, startsWith, isWhitespaceOnly, repeat } from '../utils/strings';
1414
import { HTMLDocumentRegions } from './embeddedSupport';
@@ -17,18 +17,18 @@ import * as ts from 'typescript';
1717
import { join } from 'path';
1818
import { getSemanticTokens, getSemanticTokenLegend } from './javascriptSemanticTokens';
1919

20-
const FILE_NAME = 'vscode://javascript/1'; // the same 'file' is used for all contents
21-
const TS_FILE_NAME = 'vscode://javascript/2.ts';
2220
const JS_WORD_REGEX = /(-?\d*\.\d\w*)|([^\`\~\!\@\#\%\^\&\*\(\)\-\=\+\[\{\]\}\\\|\;\:\'\"\,\.\<\>\/\?\s]+)/g;
2321

2422
let jquery_d_ts = join(__dirname, '../lib/jquery.d.ts'); // when packaged
2523
if (!ts.sys.fileExists(jquery_d_ts)) {
2624
jquery_d_ts = join(__dirname, '../../lib/jquery.d.ts'); // from source
2725
}
2826

29-
export function getJavaScriptMode(documentRegions: LanguageModelCache<HTMLDocumentRegions>): LanguageMode {
27+
export function getJavaScriptMode(documentRegions: LanguageModelCache<HTMLDocumentRegions>, id: 'javascript' | 'typescript'): LanguageMode {
3028
let jsDocuments = getLanguageModelCache<TextDocument>(10, 60, document => documentRegions.get(document).getEmbeddedDocument('javascript'));
3129

30+
const workingFile = id === 'javascript' ? 'vscode://javascript/1.js' : 'vscode://javascript/2.ts'; // the same 'file' is used for all contents
31+
3232
let compilerOptions: ts.CompilerOptions = { allowNonTsExtensions: true, allowJs: true, lib: ['lib.es6.d.ts'], target: ts.ScriptTarget.Latest, moduleResolution: ts.ModuleResolutionKind.Classic };
3333
let currentTextDocument: TextDocument;
3434
let scriptFileVersion: number = 0;
@@ -40,18 +40,18 @@ export function getJavaScriptMode(documentRegions: LanguageModelCache<HTMLDocume
4040
}
4141
const host: ts.LanguageServiceHost = {
4242
getCompilationSettings: () => compilerOptions,
43-
getScriptFileNames: () => [FILE_NAME, TS_FILE_NAME, jquery_d_ts],
44-
getScriptKind: (fileName) => fileName === TS_FILE_NAME ? ts.ScriptKind.TS : ts.ScriptKind.JS,
43+
getScriptFileNames: () => [workingFile, jquery_d_ts],
44+
getScriptKind: (fileName) => fileName.substr(fileName.length - 2) === 'ts' ? ts.ScriptKind.TS : ts.ScriptKind.JS,
4545
getScriptVersion: (fileName: string) => {
46-
if (fileName === FILE_NAME || fileName === TS_FILE_NAME) {
46+
if (fileName === workingFile) {
4747
return String(scriptFileVersion);
4848
}
4949
return '1'; // default lib an jquery.d.ts are static
5050
},
5151
getScriptSnapshot: (fileName: string) => {
5252
let text = '';
5353
if (startsWith(fileName, 'vscode:')) {
54-
if (fileName === FILE_NAME || fileName === TS_FILE_NAME) {
54+
if (fileName === workingFile) {
5555
text = currentTextDocument.getText();
5656
}
5757
} else {
@@ -72,12 +72,12 @@ export function getJavaScriptMode(documentRegions: LanguageModelCache<HTMLDocume
7272

7373
return {
7474
getId() {
75-
return 'javascript';
75+
return id;
7676
},
7777
doValidation(document: TextDocument): Diagnostic[] {
7878
updateCurrentTextDocument(document);
79-
const syntaxDiagnostics: ts.Diagnostic[] = jsLanguageService.getSyntacticDiagnostics(FILE_NAME);
80-
const semanticDiagnostics = jsLanguageService.getSemanticDiagnostics(FILE_NAME);
79+
const syntaxDiagnostics: ts.Diagnostic[] = jsLanguageService.getSyntacticDiagnostics(workingFile);
80+
const semanticDiagnostics = jsLanguageService.getSemanticDiagnostics(workingFile);
8181
return syntaxDiagnostics.concat(semanticDiagnostics).map((diag: ts.Diagnostic): Diagnostic => {
8282
return {
8383
range: convertRange(currentTextDocument, diag),
@@ -90,7 +90,7 @@ export function getJavaScriptMode(documentRegions: LanguageModelCache<HTMLDocume
9090
doComplete(document: TextDocument, position: Position): CompletionList {
9191
updateCurrentTextDocument(document);
9292
let offset = currentTextDocument.offsetAt(position);
93-
let completions = jsLanguageService.getCompletionsAtPosition(FILE_NAME, offset, { includeExternalModuleExports: false, includeInsertTextCompletions: false });
93+
let completions = jsLanguageService.getCompletionsAtPosition(workingFile, offset, { includeExternalModuleExports: false, includeInsertTextCompletions: false });
9494
if (!completions) {
9595
return { isIncomplete: false, items: [] };
9696
}
@@ -116,7 +116,7 @@ export function getJavaScriptMode(documentRegions: LanguageModelCache<HTMLDocume
116116
},
117117
doResolve(document: TextDocument, item: CompletionItem): CompletionItem {
118118
updateCurrentTextDocument(document);
119-
let details = jsLanguageService.getCompletionEntryDetails(FILE_NAME, item.data.offset, item.label, undefined, undefined, undefined);
119+
let details = jsLanguageService.getCompletionEntryDetails(workingFile, item.data.offset, item.label, undefined, undefined, undefined);
120120
if (details) {
121121
item.detail = ts.displayPartsToString(details.displayParts);
122122
item.documentation = ts.displayPartsToString(details.documentation);
@@ -126,7 +126,7 @@ export function getJavaScriptMode(documentRegions: LanguageModelCache<HTMLDocume
126126
},
127127
doHover(document: TextDocument, position: Position): Hover | null {
128128
updateCurrentTextDocument(document);
129-
let info = jsLanguageService.getQuickInfoAtPosition(FILE_NAME, currentTextDocument.offsetAt(position));
129+
let info = jsLanguageService.getQuickInfoAtPosition(workingFile, currentTextDocument.offsetAt(position));
130130
if (info) {
131131
let contents = ts.displayPartsToString(info.displayParts);
132132
return {
@@ -138,7 +138,7 @@ export function getJavaScriptMode(documentRegions: LanguageModelCache<HTMLDocume
138138
},
139139
doSignatureHelp(document: TextDocument, position: Position): SignatureHelp | null {
140140
updateCurrentTextDocument(document);
141-
let signHelp = jsLanguageService.getSignatureHelpItems(FILE_NAME, currentTextDocument.offsetAt(position), undefined);
141+
let signHelp = jsLanguageService.getSignatureHelpItems(workingFile, currentTextDocument.offsetAt(position), undefined);
142142
if (signHelp) {
143143
let ret: SignatureHelp = {
144144
activeSignature: signHelp.selectedItemIndex,
@@ -175,7 +175,7 @@ export function getJavaScriptMode(documentRegions: LanguageModelCache<HTMLDocume
175175
},
176176
findDocumentHighlight(document: TextDocument, position: Position): DocumentHighlight[] {
177177
updateCurrentTextDocument(document);
178-
const highlights = jsLanguageService.getDocumentHighlights(FILE_NAME, currentTextDocument.offsetAt(position), [FILE_NAME]);
178+
const highlights = jsLanguageService.getDocumentHighlights(workingFile, currentTextDocument.offsetAt(position), [workingFile]);
179179
const out: DocumentHighlight[] = [];
180180
for (const entry of highlights || []) {
181181
for (const highlight of entry.highlightSpans) {
@@ -189,7 +189,7 @@ export function getJavaScriptMode(documentRegions: LanguageModelCache<HTMLDocume
189189
},
190190
findDocumentSymbols(document: TextDocument): SymbolInformation[] {
191191
updateCurrentTextDocument(document);
192-
let items = jsLanguageService.getNavigationBarItems(FILE_NAME);
192+
let items = jsLanguageService.getNavigationBarItems(workingFile);
193193
if (items) {
194194
let result: SymbolInformation[] = [];
195195
let existing = Object.create(null);
@@ -225,9 +225,9 @@ export function getJavaScriptMode(documentRegions: LanguageModelCache<HTMLDocume
225225
},
226226
findDefinition(document: TextDocument, position: Position): Definition | null {
227227
updateCurrentTextDocument(document);
228-
let definition = jsLanguageService.getDefinitionAtPosition(FILE_NAME, currentTextDocument.offsetAt(position));
228+
let definition = jsLanguageService.getDefinitionAtPosition(workingFile, currentTextDocument.offsetAt(position));
229229
if (definition) {
230-
return definition.filter(d => d.fileName === FILE_NAME).map(d => {
230+
return definition.filter(d => d.fileName === workingFile).map(d => {
231231
return {
232232
uri: document.uri,
233233
range: convertRange(currentTextDocument, d.textSpan)
@@ -238,9 +238,9 @@ export function getJavaScriptMode(documentRegions: LanguageModelCache<HTMLDocume
238238
},
239239
findReferences(document: TextDocument, position: Position): Location[] {
240240
updateCurrentTextDocument(document);
241-
let references = jsLanguageService.getReferencesAtPosition(FILE_NAME, currentTextDocument.offsetAt(position));
241+
let references = jsLanguageService.getReferencesAtPosition(workingFile, currentTextDocument.offsetAt(position));
242242
if (references) {
243-
return references.filter(d => d.fileName === FILE_NAME).map(d => {
243+
return references.filter(d => d.fileName === workingFile).map(d => {
244244
return {
245245
uri: document.uri,
246246
range: convertRange(currentTextDocument, d.textSpan)
@@ -255,7 +255,7 @@ export function getJavaScriptMode(documentRegions: LanguageModelCache<HTMLDocume
255255
const parent = selectionRange.parent ? convertSelectionRange(selectionRange.parent) : undefined;
256256
return SelectionRange.create(convertRange(currentTextDocument, selectionRange.textSpan), parent);
257257
}
258-
const range = jsLanguageService.getSmartSelectionRange(FILE_NAME, currentTextDocument.offsetAt(position));
258+
const range = jsLanguageService.getSmartSelectionRange(workingFile, currentTextDocument.offsetAt(position));
259259
return convertSelectionRange(range);
260260
},
261261
format(document: TextDocument, range: Range, formatParams: FormattingOptions, settings: Settings = globalSettings): TextEdit[] {
@@ -273,7 +273,7 @@ export function getJavaScriptMode(documentRegions: LanguageModelCache<HTMLDocume
273273
end -= range.end.character;
274274
lastLineRange = Range.create(Position.create(range.end.line, 0), range.end);
275275
}
276-
let edits = jsLanguageService.getFormattingEditsForRange(FILE_NAME, start, end, formatSettings);
276+
let edits = jsLanguageService.getFormattingEditsForRange(workingFile, start, end, formatSettings);
277277
if (edits) {
278278
let result = [];
279279
for (let edit of edits) {
@@ -296,7 +296,7 @@ export function getJavaScriptMode(documentRegions: LanguageModelCache<HTMLDocume
296296
},
297297
getFoldingRanges(document: TextDocument): FoldingRange[] {
298298
updateCurrentTextDocument(document);
299-
let spans = jsLanguageService.getOutliningSpans(FILE_NAME);
299+
let spans = jsLanguageService.getOutliningSpans(workingFile);
300300
let ranges: FoldingRange[] = [];
301301
for (let span of spans) {
302302
let curr = convertRange(currentTextDocument, span.textSpan);
@@ -316,12 +316,9 @@ export function getJavaScriptMode(documentRegions: LanguageModelCache<HTMLDocume
316316
onDocumentRemoved(document: TextDocument) {
317317
jsDocuments.onDocumentRemoved(document);
318318
},
319-
getSemanticTokens(document: TextDocument, ranges: Range[] | undefined): number[] {
319+
getSemanticTokens(document: TextDocument): SemanticTokenData[] {
320320
updateCurrentTextDocument(document);
321-
if (!ranges) {
322-
ranges = [Range.create(Position.create(0, 0), document.positionAt(document.getText().length))];
323-
}
324-
return getSemanticTokens(jsLanguageService, currentTextDocument, TS_FILE_NAME, ranges);
321+
return getSemanticTokens(jsLanguageService, currentTextDocument, workingFile);
325322
},
326323
getSemanticTokenLegend(): { types: string[], modifiers: string[] } {
327324
return getSemanticTokenLegend();

‎extensions/html-language-features/server/src/modes/javascriptSemanticTokens.ts‎

Lines changed: 4 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -3,22 +3,14 @@
33
* Licensed under the MIT License. See License.txt in the project root for license information.
44
*--------------------------------------------------------------------------------------------*/
55

6-
import { TextDocument, Range } from './languageModes';
6+
import { TextDocument, SemanticTokenData } from './languageModes';
77
import * as ts from 'typescript';
88

9-
type SemanticTokenData = { offset: number, length: number, typeIdx: number, modifierSet: number };
109

11-
export function getSemanticTokens(jsLanguageService: ts.LanguageService, currentTextDocument: TextDocument, fileName: string, ranges: Range[]) {
10+
export function getSemanticTokens(jsLanguageService: ts.LanguageService, currentTextDocument: TextDocument, fileName: string): SemanticTokenData[] {
1211
//https://ts-ast-viewer.com/#code/AQ0g2CmAuwGbALzAJwG4BQZQGNwEMBnQ4AQQEYBmYAb2C22zgEtJwATJVTRxgcwD27AQAp8AGmAAjAJS0A9POB8+7NQ168oscAJz5wANXwAnLug2bsJmAFcTAO2XAA1MHyvgu-UdOeWbOw8ViAAvpagocBAA
1312

1413
let resultTokens: SemanticTokenData[] = [];
15-
// const tokens = jsLanguageService.getSemanticClassifications(fileName, { start: 0, length: currentTextDocument.getText().length });
16-
// for (let token of tokens) {
17-
// const typeIdx = tokenFromClassificationMapping[token.classificationType];
18-
// if (typeIdx !== undefined) {
19-
// resultTokens.push({ offset: token.textSpan.start, length: token.textSpan.length, typeIdx, modifierSet: 0 });
20-
// }
21-
// }
2214

2315
const program = jsLanguageService.getProgram();
2416
if (program) {
@@ -46,7 +38,7 @@ export function getSemanticTokens(jsLanguageService: ts.LanguageService, current
4638
modifierSet |= TokenModifier.async;
4739
}
4840
if (typeIdx !== undefined) {
49-
resultTokens.push({ offset: node.getStart(), length: node.getWidth(), typeIdx, modifierSet });
41+
resultTokens.push({ start: currentTextDocument.positionAt(node.getStart()), length: node.getWidth(), typeIdx, modifierSet });
5042
}
5143
}
5244
}
@@ -60,41 +52,7 @@ export function getSemanticTokens(jsLanguageService: ts.LanguageService, current
6052
}
6153
}
6254

63-
64-
resultTokens = resultTokens.sort((d1, d2) => d1.offset - d2.offset);
65-
const offsetRanges = ranges.map(r => ({ startOffset: currentTextDocument.offsetAt(r.start), endOffset: currentTextDocument.offsetAt(r.end) })).sort((d1, d2) => d1.startOffset - d2.startOffset);
66-
67-
let rangeIndex = 0;
68-
let currRange = offsetRanges[rangeIndex++];
69-
70-
let prefLine = 0;
71-
let prevChar = 0;
72-
73-
let encodedResult: number[] = [];
74-
75-
for (let k = 0; k < resultTokens.length && currRange; k++) {
76-
const curr = resultTokens[k];
77-
if (currRange.startOffset <= curr.offset && curr.offset + curr.length <= currRange.endOffset) {
78-
// token inside a range
79-
80-
const startPos = currentTextDocument.positionAt(curr.offset);
81-
if (prefLine !== startPos.line) {
82-
prevChar = 0;
83-
}
84-
encodedResult.push(startPos.line - prefLine); // line delta
85-
encodedResult.push(startPos.character - prevChar); // line delta
86-
encodedResult.push(curr.length); // length
87-
encodedResult.push(curr.typeIdx); // tokenType
88-
encodedResult.push(curr.modifierSet); // tokenModifier
89-
90-
prefLine = startPos.line;
91-
prevChar = startPos.character;
92-
93-
} else if (currRange.endOffset >= curr.offset) {
94-
currRange = offsetRanges[rangeIndex++];
95-
}
96-
}
97-
return encodedResult;
55+
return resultTokens;
9856
}
9957

10058

@@ -127,16 +85,6 @@ enum TokenModifier {
12785
'async' = 0x04,
12886
}
12987

130-
// const tokenFromClassificationMapping: { [name: string]: TokenType } = {
131-
// [ts.ClassificationTypeNames.className]: TokenType.class,
132-
// [ts.ClassificationTypeNames.enumName]: TokenType.enum,
133-
// [ts.ClassificationTypeNames.interfaceName]: TokenType.interface,
134-
// [ts.ClassificationTypeNames.moduleName]: TokenType.namespace,
135-
// [ts.ClassificationTypeNames.typeParameterName]: TokenType.parameterType,
136-
// [ts.ClassificationTypeNames.typeAliasName]: TokenType.type,
137-
// [ts.ClassificationTypeNames.parameterName]: TokenType.parameter
138-
// };
139-
14088
const tokenFromDeclarationMapping: { [name: string]: TokenType } = {
14189
[ts.SyntaxKind.VariableDeclaration]: TokenType.variable,
14290
[ts.SyntaxKind.Parameter]: TokenType.parameter,

‎extensions/html-language-features/server/src/modes/languageModes.ts‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,13 @@ export interface Workspace {
3131
readonly folders: WorkspaceFolder[];
3232
}
3333

34+
export interface SemanticTokenData {
35+
start: Position;
36+
length: number;
37+
typeIdx: number;
38+
modifierSet: number;
39+
}
40+
3441
export interface LanguageMode {
3542
getId(): string;
3643
getSelectionRange?: (document: TextDocument, position: Position) => SelectionRange;
@@ -52,7 +59,7 @@ export interface LanguageMode {
5259
findMatchingTagPosition?: (document: TextDocument, position: Position) => Position | null;
5360
getFoldingRanges?: (document: TextDocument) => FoldingRange[];
5461
onDocumentRemoved(document: TextDocument): void;
55-
getSemanticTokens?(document: TextDocument, ranges: Range[] | undefined): number[];
62+
getSemanticTokens?(document: TextDocument): SemanticTokenData[];
5663
getSemanticTokenLegend?(): { types: string[], modifiers: string[] };
5764
dispose(): void;
5865
}
@@ -87,7 +94,8 @@ export function getLanguageModes(supportedLanguages: { [languageId: string]: boo
8794
modes['css'] = getCSSMode(cssLanguageService, documentRegions, workspace);
8895
}
8996
if (supportedLanguages['javascript']) {
90-
modes['javascript'] = getJavaScriptMode(documentRegions);
97+
modes['javascript'] = getJavaScriptMode(documentRegions, 'javascript');
98+
modes['typescript'] = getJavaScriptMode(documentRegions, 'typescript');
9199
}
92100
return {
93101
getModeAtPosition(document: TextDocument, position: Position): LanguageMode | undefined {

0 commit comments

Comments
 (0)