Skip to content

Commit f22940a

Browse files
aeschliaeschli
authored
Merge pull request microsoft#58703 from Microsoft/aeschli/tokenizersupport-initialization
ITokenizationRegistry to support TokenizationSupport promises
2 parents b2e70a2 + 51ef4de commit f22940a

7 files changed

Lines changed: 66 additions & 30 deletions

File tree

‎src/vs/editor/common/modes.ts‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1186,12 +1186,23 @@ export interface ITokenizationRegistry {
11861186
*/
11871187
register(language: string, support: ITokenizationSupport): IDisposable;
11881188

1189+
/**
1190+
* Register a promise for a tokenization support.
1191+
*/
1192+
registerPromise(language: string, promise: Thenable<ITokenizationSupport>): Thenable<IDisposable>;
1193+
11891194
/**
11901195
* Get the tokenization support for a language.
11911196
* Returns null if not found.
11921197
*/
11931198
get(language: string): ITokenizationSupport;
11941199

1200+
/**
1201+
* Get the promise of a tokenization support for a language.
1202+
* `null` is returned if no support is available and no promise for the support has been registered yet.
1203+
*/
1204+
getPromise(language: string): Thenable<ITokenizationSupport>;
1205+
11951206
/**
11961207
* Set the new color map that all tokens will use in their ColorId binary encoded bits for foreground and background.
11971208
*/

‎src/vs/editor/common/modes/textToHtmlTokenizer.ts‎

Lines changed: 10 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -5,13 +5,19 @@
55
'use strict';
66

77
import * as strings from 'vs/base/common/strings';
8-
import { IState, ITokenizationSupport, TokenizationRegistry, LanguageId } from 'vs/editor/common/modes';
9-
import { NULL_STATE, nullTokenize2 } from 'vs/editor/common/modes/nullMode';
8+
import { ITokenizationSupport, IState, LanguageId } from 'vs/editor/common/modes';
109
import { LineTokens, IViewLineTokens } from 'vs/editor/common/core/lineTokens';
1110
import { CharCode } from 'vs/base/common/charCode';
11+
import { NULL_STATE, nullTokenize2 } from 'vs/editor/common/modes/nullMode';
12+
13+
const fallback = {
14+
getInitialState: () => NULL_STATE,
15+
tokenize: undefined,
16+
tokenize2: (buffer: string, state: IState, deltaOffset: number) => nullTokenize2(LanguageId.Null, buffer, state, deltaOffset)
17+
};
1218

13-
export function tokenizeToString(text: string, languageId: string): string {
14-
return _tokenizeToString(text, _getSafeTokenizationSupport(languageId));
19+
export function tokenizeToString(text: string, tokenizationSupport: ITokenizationSupport = fallback): string {
20+
return _tokenizeToString(text, tokenizationSupport);
1521
}
1622

1723
export function tokenizeLineToHTML(text: string, viewLineTokens: IViewLineTokens, colorMap: string[], startOffset: number, endOffset: number, tabSize: number): string {
@@ -83,18 +89,6 @@ export function tokenizeLineToHTML(text: string, viewLineTokens: IViewLineTokens
8389
return result;
8490
}
8591

86-
function _getSafeTokenizationSupport(languageId: string): ITokenizationSupport {
87-
let tokenizationSupport = TokenizationRegistry.get(languageId);
88-
if (tokenizationSupport) {
89-
return tokenizationSupport;
90-
}
91-
return {
92-
getInitialState: () => NULL_STATE,
93-
tokenize: undefined,
94-
tokenize2: (buffer: string, state: IState, deltaOffset: number) => nullTokenize2(LanguageId.Null, buffer, state, deltaOffset)
95-
};
96-
}
97-
9892
function _tokenizeToString(text: string, tokenizationSupport: ITokenizationSupport): string {
9993
let result = `<div class="monaco-tokenized-source">`;
10094
let lines = text.split(/\r\n|\r|\n/);

‎src/vs/editor/common/modes/tokenizationRegistry.ts‎

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,12 @@ import { IDisposable, toDisposable } from 'vs/base/common/lifecycle';
88
import { Event, Emitter } from 'vs/base/common/event';
99
import { ColorId, ITokenizationRegistry, ITokenizationSupport, ITokenizationSupportChangedEvent } from 'vs/editor/common/modes';
1010
import { Color } from 'vs/base/common/color';
11+
import { TPromise } from 'vs/base/common/winjs.base';
1112

1213
export class TokenizationRegistryImpl implements ITokenizationRegistry {
1314

1415
private _map: { [language: string]: ITokenizationSupport };
16+
private _promises: { [language: string]: Thenable<IDisposable> };
1517

1618
private readonly _onDidChange: Emitter<ITokenizationSupportChangedEvent> = new Emitter<ITokenizationSupportChangedEvent>();
1719
public readonly onDidChange: Event<ITokenizationSupportChangedEvent> = this._onDidChange.event;
@@ -20,6 +22,7 @@ export class TokenizationRegistryImpl implements ITokenizationRegistry {
2022

2123
constructor() {
2224
this._map = Object.create(null);
25+
this._promises = Object.create(null);
2326
this._colorMap = null;
2427
}
2528

@@ -30,7 +33,7 @@ export class TokenizationRegistryImpl implements ITokenizationRegistry {
3033
});
3134
}
3235

33-
public register(language: string, support: ITokenizationSupport): IDisposable {
36+
public register(language: string, support: ITokenizationSupport) {
3437
this._map[language] = support;
3538
this.fire([language]);
3639
return toDisposable(() => {
@@ -42,6 +45,26 @@ export class TokenizationRegistryImpl implements ITokenizationRegistry {
4245
});
4346
}
4447

48+
public registerPromise(language: string, supportPromise: Thenable<ITokenizationSupport>): Thenable<IDisposable> {
49+
const promise = this._promises[language] = supportPromise.then(support => {
50+
delete this._promises[language];
51+
return this.register(language, support);
52+
});
53+
return promise;
54+
}
55+
56+
public getPromise(language: string): Thenable<ITokenizationSupport> {
57+
const support = this.get(language);
58+
if (support) {
59+
return TPromise.as(support);
60+
}
61+
const promise = this._promises[language];
62+
if (promise) {
63+
return promise.then(_ => this.get(language));
64+
}
65+
return null;
66+
}
67+
4568
public get(language: string): ITokenizationSupport {
4669
return (this._map[language] || null);
4770
}

‎src/vs/editor/contrib/markdown/markdownRenderer.ts‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ import { ICodeEditor } from 'vs/editor/browser/editorBrowser';
1717
import { optional } from 'vs/platform/instantiation/common/instantiation';
1818
import { Event, Emitter } from 'vs/base/common/event';
1919
import { IDisposable, dispose } from 'vs/base/common/lifecycle';
20+
import { TokenizationRegistry } from 'vs/editor/common/modes';
2021

2122
export interface IMarkdownRenderResult extends IDisposable {
2223
element: HTMLElement;
@@ -45,7 +46,11 @@ export class MarkdownRenderer {
4546
: this._editor.getModel().getLanguageIdentifier().language;
4647

4748
return this._modeService.getOrCreateMode(modeId).then(_ => {
48-
return tokenizeToString(value, modeId);
49+
const promise = TokenizationRegistry.getPromise(modeId);
50+
if (promise) {
51+
return promise.then(support => tokenizeToString(value, support));
52+
}
53+
return tokenizeToString(value, null);
4954
}).then(code => {
5055
return `<span style="font-family: ${this._editor.getConfiguration().fontInfo.fontFamily}">${code}</span>`;
5156
});

‎src/vs/editor/test/common/modes/textToHtmlTokenizer.test.ts‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,9 @@ suite('Editor Modes - textToHtmlTokenizer', () => {
1919

2020
test('TextToHtmlTokenizer 1', () => {
2121
let mode = new Mode();
22+
let support = TokenizationRegistry.get(mode.getId());
2223

23-
let actual = tokenizeToString('.abc..def...gh', mode.getId());
24+
let actual = tokenizeToString('.abc..def...gh', support);
2425
let expected = [
2526
{ className: 'mtk7', text: '.' },
2627
{ className: 'mtk9', text: 'abc' },
@@ -38,8 +39,9 @@ suite('Editor Modes - textToHtmlTokenizer', () => {
3839

3940
test('TextToHtmlTokenizer 2', () => {
4041
let mode = new Mode();
42+
let support = TokenizationRegistry.get(mode.getId());
4143

42-
let actual = tokenizeToString('.abc..def...gh\n.abc..def...gh', mode.getId());
44+
let actual = tokenizeToString('.abc..def...gh\n.abc..def...gh', support);
4345
let expected1 = [
4446
{ className: 'mtk7', text: '.' },
4547
{ className: 'mtk9', text: 'abc' },

‎src/vs/workbench/parts/update/electron-browser/releaseNotesEditor.ts‎

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ import { OS } from 'vs/base/common/platform';
1111
import { URI } from 'vs/base/common/uri';
1212
import { TPromise } from 'vs/base/common/winjs.base';
1313
import { asText } from 'vs/base/node/request';
14-
import { IMode, TokenizationRegistry } from 'vs/editor/common/modes';
14+
import { TokenizationRegistry, ITokenizationSupport } from 'vs/editor/common/modes';
1515
import { generateTokensCSSForColorMap } from 'vs/editor/common/modes/supports/tokenization';
1616
import { tokenizeToString } from 'vs/editor/common/modes/textToHtmlTokenizer';
1717
import { IModeService } from 'vs/editor/common/services/modeService';
@@ -192,21 +192,18 @@ export class ReleaseNotesManager {
192192
}
193193

194194
private async getRenderer(text: string) {
195-
const result: TPromise<IMode>[] = [];
195+
let result: TPromise<ITokenizationSupport>[] = [];
196196
const renderer = new marked.Renderer();
197197
renderer.code = (code, lang) => {
198198
const modeId = this._modeService.getModeIdForLanguageName(lang);
199-
result.push(this._modeService.getOrCreateMode(modeId));
199+
result.push(this._modeService.getOrCreateMode(modeId).then(_ => TokenizationRegistry.getPromise(modeId)));
200200
return '';
201201
};
202202

203203
marked(text, { renderer });
204204
await TPromise.join(result);
205205

206-
renderer.code = (code, lang) => {
207-
const modeId = this._modeService.getModeIdForLanguageName(lang);
208-
return `<code>${tokenizeToString(code, modeId)}</code>`;
209-
};
206+
renderer.code = (code, lang) => `<code>${tokenizeToString(code, TokenizationRegistry.get(lang))}</code>`;
210207
return renderer;
211208
}
212209
}

‎src/vs/workbench/services/textMate/electron-browser/TMSyntax.ts‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -397,9 +397,13 @@ export class TextMateService implements ITextMateService {
397397
}
398398

399399
private registerDefinition(modeId: string): void {
400-
this._createGrammar(modeId).then((r) => {
401-
TokenizationRegistry.register(modeId, new TMTokenization(this._scopeRegistry, r.languageId, r.grammar, r.initialState, r.containsEmbeddedLanguages, this._notificationService));
402-
}, onUnexpectedError);
400+
const promise = this._createGrammar(modeId).then((r) => {
401+
return new TMTokenization(this._scopeRegistry, r.languageId, r.grammar, r.initialState, r.containsEmbeddedLanguages, this._notificationService);
402+
}, e => {
403+
onUnexpectedError(e);
404+
return null;
405+
});
406+
TokenizationRegistry.registerPromise(modeId, promise);
403407
}
404408
}
405409

0 commit comments

Comments
 (0)