Ensure @rematch and nextEmbedded can be used together in a Monarch grammar.

bolinfest · bolinfest · commit b9f5b9eeb21c · 2020-04-20T22:48:18.000-07:00
Similar to the repro case I created for microsoft#90266, I created a local playground with the following: ```js const DEMO_LANG_ID = "demo"; const SQL_QUERY_START = "(SELECT|INSERT|UPDATE|DELETE|CREATE|REPLACE|ALTER|WITH)"; const languageConfiguration = { tokenizer: { root: [ [ `(\"\"\")${SQL_QUERY_START}`, [ { "token": "string.quote", }, { token: "@rematch", next: "@endStringWithSQL", nextEmbedded: "sql", }, ] ], [ /(""")$/, [ { token: "string.quote", next: "@maybeStringIsSQL", }, ] ], ], maybeStringIsSQL: [ [ /(.*)/, { cases: { [`${SQL_QUERY_START}\\b.*`]: { token: "@rematch", next: "@endStringWithSQL", nextEmbedded: "sql", }, "@default": { token: "@rematch", switchTo: "@endDblDocString", }, } } ], ], endDblDocString: [ [ "[^\"]+", "string" ], [ "\\\\\"", "string" ], [ "\"\"\"", "string", "@popall" ], [ "\"", "string" ] ], endStringWithSQL: [ [ /"""/, { token: "string.quote", next: "@popall", nextEmbedded: "@pop", }, ] ], }, }; monaco.languages.register({ id: DEMO_LANG_ID, extensions: ['.example'], }); monaco.languages.setMonarchTokensProvider(DEMO_LANG_ID, languageConfiguration); const value = `\ mysql_query("""SELECT * FROM table_name WHERE ds = '<DATEID>'""") mysql_query(""" SELECT * FROM table_name WHERE ds = '<DATEID>' """) `; var editor = monaco.editor.create(document.getElementById("container"), { value, language: DEMO_LANG_ID, lineNumbers: "off", roundedSelection: false, scrollBeyondLastLine: false, readOnly: false, theme: "vs-dark", }); ``` Without my change to monarchLexer.ts, I get this error in the console: > errors.ts:26 Uncaught Error: demo: cannot pop embedded mode if not inside one But with my change, I see a Python-esque multiline string literal with SQL syntax highlighting within it.
diff --git a/src/vs/editor/standalone/common/monarch/monarchLexer.ts b/src/vs/editor/standalone/common/monarch/monarchLexer.ts
@@ -740,6 +740,24 @@ export class MonarchTokenizer implements modes.ITokenizationSupport {
 				throw monarchCommon.createError(this._lexer, 'lexer rule has no well-defined action in rule: ' + this._safeRuleName(rule));
 			}
 
+			const computeNewStateForEmbeddedMode = (enteringEmbeddedMode: string) => {
+				// substitute language alias to known modes to support syntax highlighting
+				let enteringEmbeddedModeId = this._modeService.getModeIdForLanguageName(enteringEmbeddedMode);
+				if (enteringEmbeddedModeId) {
+					enteringEmbeddedMode = enteringEmbeddedModeId;
+				}
+
+				const embeddedModeData = this._getNestedEmbeddedModeData(enteringEmbeddedMode);
+
+				if (pos < lineLength) {
+					// there is content from the embedded mode on this line
+					const restOfLine = line.substr(pos);
+					return this._nestedTokenize(restOfLine, MonarchLineStateFactory.create(stack, embeddedModeData), offsetDelta + pos, tokensCollector);
+				} else {
+					return MonarchLineStateFactory.create(stack, embeddedModeData);
+				}
+			};
+
 			// is the result a group match?
 			if (Array.isArray(result)) {
 				if (groupMatching && groupMatching.groups.length > 0) {
@@ -780,6 +798,12 @@ export class MonarchTokenizer implements modes.ITokenizationSupport {
 					matched = '';  // better set the next state too..
 					matches = null;
 					result = '';
+
+					// Even though `@rematch` was specified, if `nextEmbedded` also specified,
+					// a state transition should occur.
+					if (enteringEmbeddedMode !== null) {
+						return computeNewStateForEmbeddedMode(enteringEmbeddedMode);
+					}
 				}
 
 				// check progress
@@ -810,21 +834,7 @@ export class MonarchTokenizer implements modes.ITokenizationSupport {
 			}
 
 			if (enteringEmbeddedMode !== null) {
-				// substitute language alias to known modes to support syntax highlighting
-				let enteringEmbeddedModeId = this._modeService.getModeIdForLanguageName(enteringEmbeddedMode);
-				if (enteringEmbeddedModeId) {
-					enteringEmbeddedMode = enteringEmbeddedModeId;
-				}
-
-				let embeddedModeData = this._getNestedEmbeddedModeData(enteringEmbeddedMode);
-
-				if (pos < lineLength) {
-					// there is content from the embedded mode on this line
-					let restOfLine = line.substr(pos);
-					return this._nestedTokenize(restOfLine, MonarchLineStateFactory.create(stack, embeddedModeData), offsetDelta + pos, tokensCollector);
-				} else {
-					return MonarchLineStateFactory.create(stack, embeddedModeData);
-				}
+				return computeNewStateForEmbeddedMode(enteringEmbeddedMode);
 			}
 		}
 
