Fixes microsoft#8712: Bail out of tokenization if too many states are pushed and not popped

alexdima · alexdima · commit f7d23a3aa13c · 2016-07-15T10:45:29.000+03:00
diff --git a/src/vs/editor/node/textMate/TMSyntax.ts b/src/vs/editor/node/textMate/TMSyntax.ts
@@ -12,7 +12,7 @@ import {ILineTokens, IMode, ITokenizationSupport} from 'vs/editor/common/modes';
 import {TMState} from 'vs/editor/common/modes/TMState';
 import {LineTokens, Token} from 'vs/editor/common/modes/supports';
 import {IModeService} from 'vs/editor/common/services/modeService';
-import {IGrammar, Registry} from 'vscode-textmate';
+import {IGrammar, Registry, StackElement} from 'vscode-textmate';
 import {ModeTransition} from 'vs/editor/common/core/modeTransition';
 import {IConfigurationService} from 'vs/platform/configuration/common/configuration';
 
@@ -239,6 +239,15 @@ export class TMTokenDecodeData {
 	}
 }
 
+function depth(stackElement: StackElement): number {
+	let result = 0;
+	while (stackElement) {
+		result++;
+		stackElement = stackElement._parent;
+	}
+	return result;
+}
+
 class Tokenizer {
 	private _grammar: IGrammar;
 	private _modeId: string;
@@ -251,7 +260,9 @@ class Tokenizer {
 	}
 
 	public tokenize(line: string, state: TMState, offsetDelta: number = 0, stopAtOffset?: number): ILineTokens {
-		if (line.length >= 20000) {
+		// Do not attempt to tokenize if a line has over 20k
+		// or if the rule stack contains more than 30 rules (indicator of broken grammar that forgets to pop rules)
+		if (line.length >= 20000 || depth(state.getRuleStack()) > 30) {
 			return new LineTokens(
 				[new Token(offsetDelta, '')],
 				[new ModeTransition(offsetDelta, state.getMode())],
diff --git a/src/vs/editor/node/textMate/vscode-textmate.d.ts b/src/vs/editor/node/textMate/vscode-textmate.d.ts
@@ -72,6 +72,7 @@ export interface IToken {
  */
 export interface StackElement {
 	_stackElementBrand: void;
+	_parent: StackElement;
 
 	equals(other:StackElement): boolean;
 }

Original file line number	Diff line number	Diff line change
`@@ -72,6 +72,7 @@ export interface IToken {`
`72`	`72`	`*/`
`73`	`73`	`export interface StackElement {`
`74`	`74`	`_stackElementBrand: void;`
	`75`	`+ _parent: StackElement;`
`75`	`76`
`76`	`77`	`equals(other:StackElement): boolean;`
`77`	`78`	`}`