Print debug information on text mate stack overflow

aeschli · aeschli · commit bd6b32b43274 · 2016-12-28T21:16:43.000-04:00
diff --git a/src/vs/editor/node/textMate/TMSyntax.ts b/src/vs/editor/node/textMate/TMSyntax.ts
@@ -446,24 +446,44 @@ class Tokenizer {
 	private _grammar: IGrammar;
 	private _modeId: string;
 	private _decodeMap: DecodeMap;
+	private _stackOverflowReported: boolean;
 
 	constructor(languageRegistration: TMLanguageRegistration, modeId: string, grammar: IGrammar) {
 		this._modeId = modeId;
 		this._grammar = grammar;
 		this._decodeMap = new DecodeMap(languageRegistration);
+		this._stackOverflowReported = false;
 	}
 
 	public tokenize(line: string, state: TMState, offsetDelta: number = 0, stopAtOffset?: number): ILineTokens {
+
 		// Do not attempt to tokenize if a line has over 20k
+		if (line.length >= 20000) {
+			console.log(`Line (${line.substr(0, 15)}...): longer than 20k characters, tokenization skipped.`);
+			return new RawLineTokens(
+				[new Token(offsetDelta, '')],
+				[new ModeTransition(offsetDelta, this._modeId)],
+				offsetDelta,
+				state
+			);
+		}
+
 		// or if the rule stack contains more than 100 rules (indicator of broken grammar that forgets to pop rules)
-		if (line.length >= 20000 || depth(state.ruleStack) > 100) {
+		if (depth(state.ruleStack) > 100) {
+			if (!this._stackOverflowReported) {
+				// don't report again unless a good state has been reached again.
+				console.log(`Line (${line.substr(0, 15)}...): stack deeper than 100: tokenization stopped.`, printRuleStack(state.ruleStack));
+				this._stackOverflowReported = true;
+			}
 			return new RawLineTokens(
 				[new Token(offsetDelta, '')],
 				[new ModeTransition(offsetDelta, this._modeId)],
 				offsetDelta,
 				state
 			);
 		}
+		this._stackOverflowReported = false;
+
 		let textMateResult = this._grammar.tokenizeLine(line, state.ruleStack);
 
 		let endState: TMState;
@@ -478,6 +498,16 @@ class Tokenizer {
 	}
 }
 
+// TODO: replace with something like ruleStack.toDebugString
+function printRuleStack(ruleStack: StackElement): string[] {
+	let scopes = [];
+	while (ruleStack) {
+		scopes.push(ruleStack['_scopeName']);
+		ruleStack = ruleStack._parent;
+	}
+	return scopes;
+}
+
 export function decodeTextMateTokens(topLevelModeId: string, decodeMap: DecodeMap, line: string, offsetDelta: number, resultTokens: IToken[], resultState: TMState): RawLineTokens {
 
 	// Create the result early and fill in the tokens later
