Skip to content

Commit 6e61a29

Browse files
committed
vscode-textmate@3.0.0
1 parent 941812f commit 6e61a29

5 files changed

Lines changed: 179 additions & 96 deletions

File tree

npm-shrinkwrap.json

Lines changed: 3 additions & 3 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

package.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@
3333
"pty.js": "https://github.com/Tyriar/pty.js/tarball/c75c2dcb6dcad83b0cb3ef2ae42d0448fb912642",
3434
"semver": "4.3.6",
3535
"vscode-debugprotocol": "1.15.0",
36-
"vscode-textmate": "2.3.2",
36+
"vscode-textmate": "3.0.0",
3737
"winreg": "1.2.0",
3838
"xterm": "git+https://github.com/Tyriar/xterm.js.git#vscode-release/1.9",
3939
"yauzl": "2.3.1"

src/typings/vscode-textmate.d.ts

Lines changed: 166 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,166 @@
1+
/*---------------------------------------------------------------------------------------------
2+
* Copyright (c) Microsoft Corporation. All rights reserved.
3+
* Licensed under the MIT License. See License.txt in the project root for license information.
4+
*--------------------------------------------------------------------------------------------*/
5+
6+
declare module "vscode-textmate" {
7+
/**
8+
* A single theme setting.
9+
*/
10+
export interface IRawThemeSetting {
11+
readonly name?: string;
12+
readonly scope?: string | string[];
13+
readonly settings: {
14+
readonly fontStyle?: string;
15+
readonly foreground?: string;
16+
readonly background?: string;
17+
};
18+
}
19+
/**
20+
* A TextMate theme.
21+
*/
22+
export interface IRawTheme {
23+
readonly name?: string;
24+
readonly settings: IRawThemeSetting[];
25+
}
26+
/**
27+
* A registry helper that can locate grammar file paths given scope names.
28+
*/
29+
export interface RegistryOptions {
30+
theme?: IRawTheme;
31+
getFilePath(scopeName: string): string;
32+
getInjections?(scopeName: string): string[];
33+
}
34+
/**
35+
* A map from scope name to a language id. Please do not use language id 0.
36+
*/
37+
export interface IEmbeddedLanguagesMap {
38+
[scopeName: string]: number;
39+
}
40+
export const enum StandardTokenType {
41+
Other = 0,
42+
Comment = 1,
43+
String = 2,
44+
RegEx = 4,
45+
}
46+
/**
47+
* The registry that will hold all grammars.
48+
*/
49+
export class Registry {
50+
private readonly _locator;
51+
private readonly _syncRegistry;
52+
constructor(locator?: RegistryOptions);
53+
/**
54+
* Change the theme. Once called, no previous `ruleStack` should be used anymore.
55+
*/
56+
setTheme(theme: IRawTheme): void;
57+
/**
58+
* Returns a lookup array for color ids.
59+
*/
60+
getColorMap(): string[];
61+
/**
62+
* Load the grammar for `scopeName` and all referenced included grammars asynchronously.
63+
* Please do not use language id 0.
64+
*/
65+
loadGrammarWithEmbeddedLanguages(initialScopeName: string, initialLanguage: number, embeddedLanguages: IEmbeddedLanguagesMap, callback: (err: any, grammar: IGrammar) => void): void;
66+
/**
67+
* Load the grammar for `scopeName` and all referenced included grammars asynchronously.
68+
*/
69+
loadGrammar(initialScopeName: string, callback: (err: any, grammar: IGrammar) => void): void;
70+
private _loadGrammar(initialScopeName, callback);
71+
/**
72+
* Load the grammar at `path` synchronously.
73+
*/
74+
loadGrammarFromPathSync(path: string, initialLanguage?: number, embeddedLanguages?: IEmbeddedLanguagesMap): IGrammar;
75+
/**
76+
* Get the grammar for `scopeName`. The grammar must first be created via `loadGrammar` or `loadGrammarFromPathSync`.
77+
*/
78+
grammarForScopeName(scopeName: string, initialLanguage?: number, embeddedLanguages?: IEmbeddedLanguagesMap): IGrammar;
79+
}
80+
/**
81+
* A grammar
82+
*/
83+
export interface IGrammar {
84+
/**
85+
* Tokenize `lineText` using previous line state `prevState`.
86+
*/
87+
tokenizeLine(lineText: string, prevState: StackElement): ITokenizeLineResult;
88+
/**
89+
* Tokenize `lineText` using previous line state `prevState`.
90+
* The result contains the tokens in binary format, resolved with the following information:
91+
* - language
92+
* - token type (regex, string, comment, other)
93+
* - font style
94+
* - foreground color
95+
* - background color
96+
* e.g. for getting the languageId: `(metadata & MetadataConsts.LANGUAGEID_MASK) >>> MetadataConsts.LANGUAGEID_OFFSET`
97+
*/
98+
tokenizeLine2(lineText: string, prevState: StackElement): ITokenizeLineResult2;
99+
}
100+
export interface ITokenizeLineResult {
101+
readonly tokens: IToken[];
102+
/**
103+
* The `prevState` to be passed on to the next line tokenization.
104+
*/
105+
readonly ruleStack: StackElement;
106+
}
107+
/**
108+
* Helpers to manage the "collapsed" metadata of an entire StackElement stack.
109+
* The following assumptions have been made:
110+
* - languageId < 256 => needs 8 bits
111+
* - unique color count < 512 => needs 9 bits
112+
*
113+
* The binary format is:
114+
* - -------------------------------------------
115+
* 3322 2222 2222 1111 1111 1100 0000 0000
116+
* 1098 7654 3210 9876 5432 1098 7654 3210
117+
* - -------------------------------------------
118+
* xxxx xxxx xxxx xxxx xxxx xxxx xxxx xxxx
119+
* bbbb bbbb bfff ffff ffFF FTTT LLLL LLLL
120+
* - -------------------------------------------
121+
* - L = LanguageId (8 bits)
122+
* - T = StandardTokenType (3 bits)
123+
* - F = FontStyle (3 bits)
124+
* - f = foreground color (9 bits)
125+
* - b = background color (9 bits)
126+
*/
127+
export const enum MetadataConsts {
128+
LANGUAGEID_MASK = 255,
129+
TOKEN_TYPE_MASK = 1792,
130+
FONT_STYLE_MASK = 14336,
131+
FOREGROUND_MASK = 8372224,
132+
BACKGROUND_MASK = 4286578688,
133+
LANGUAGEID_OFFSET = 0,
134+
TOKEN_TYPE_OFFSET = 8,
135+
FONT_STYLE_OFFSET = 11,
136+
FOREGROUND_OFFSET = 14,
137+
BACKGROUND_OFFSET = 23,
138+
}
139+
export interface ITokenizeLineResult2 {
140+
/**
141+
* The tokens in binary format. Each token occupies two array indices. For token i:
142+
* - at offset 2*i => startIndex
143+
* - at offset 2*i + 1 => metadata
144+
*
145+
*/
146+
readonly tokens: Uint32Array;
147+
/**
148+
* The `prevState` to be passed on to the next line tokenization.
149+
*/
150+
readonly ruleStack: StackElement;
151+
}
152+
export interface IToken {
153+
startIndex: number;
154+
readonly endIndex: number;
155+
readonly scopes: string[];
156+
}
157+
/**
158+
* **IMPORTANT** - Immutable!
159+
*/
160+
export interface StackElement {
161+
_stackElementBrand: void;
162+
readonly depth: number;
163+
equals(other: StackElement): boolean;
164+
}
165+
166+
}

src/vs/editor/node/textMate/TMSyntax.ts

Lines changed: 9 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -438,12 +438,7 @@ export class DecodeMap {
438438
}
439439

440440
function depth(stackElement: StackElement): number {
441-
let result = 0;
442-
while (stackElement) {
443-
result++;
444-
stackElement = stackElement._parent;
445-
}
446-
return result;
441+
return stackElement.depth;
447442
}
448443

449444
class Tokenizer {
@@ -502,12 +497,14 @@ class Tokenizer {
502497

503498
// TODO: replace with something like ruleStack.toDebugString
504499
function printRuleStack(ruleStack: StackElement): string[] {
505-
let scopes = [];
506-
while (ruleStack) {
507-
scopes.push(ruleStack['_scopeName']);
508-
ruleStack = ruleStack._parent;
509-
}
510-
return scopes;
500+
// TODO@tokenization
501+
throw new Error('Not implemented');
502+
// let scopes = [];
503+
// while (ruleStack) {
504+
// scopes.push(ruleStack['_scopeName']);
505+
// ruleStack = ruleStack._parent;
506+
// }
507+
// return scopes;
511508
}
512509

513510
export function decodeTextMateTokens(topLevelModeId: string, decodeMap: DecodeMap, line: string, offsetDelta: number, resultTokens: IToken[], resultState: TMState): RawLineTokens {

src/vs/editor/node/textMate/vscode-textmate.d.ts

Lines changed: 0 additions & 80 deletions
This file was deleted.

0 commit comments

Comments
 (0)