Fixes microsoft#86303: Fix adjusting of tokens after the deleted range

alexdima · alexdima · commit 29f4c535cbbc · 2019-12-05T11:49:07.000+01:00
diff --git a/src/vs/editor/common/model/tokensStore.ts b/src/vs/editor/common/model/tokensStore.ts
@@ -293,8 +293,13 @@ export class SparseEncodedTokens implements IEncodedTokens {
 			} else if (tokenDeltaLine === endDeltaLine && tokenStartCharacter >= endCharacter) {
 				// 4. (continued) The token starts after the deletion range, on the last line where a deletion occurs
 				tokenDeltaLine -= deletedLineCount;
-				tokenStartCharacter -= endCharacter;
-				tokenEndCharacter -= endCharacter;
+				if (deletedLineCount === 0) {
+					tokenStartCharacter -= (endCharacter - startCharacter);
+					tokenEndCharacter -= (endCharacter - startCharacter);
+				} else {
+					tokenStartCharacter -= endCharacter;
+					tokenEndCharacter -= endCharacter;
+				}
 			} else {
 				throw new Error(`Not possible!`);
 			}
diff --git a/src/vs/editor/test/common/model/tokensStore.test.ts b/src/vs/editor/test/common/model/tokensStore.test.ts
@@ -0,0 +1,119 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import * as assert from 'assert';
+import { MultilineTokens2, SparseEncodedTokens } from 'vs/editor/common/model/tokensStore';
+import { Range } from 'vs/editor/common/core/range';
+import { TextModel } from 'vs/editor/common/model/textModel';
+import { IIdentifiedSingleEditOperation } from 'vs/editor/common/model';
+import { MetadataConsts, TokenMetadata } from 'vs/editor/common/modes';
+
+suite('TokensStore', () => {
+
+	const SEMANTIC_COLOR = 5;
+
+	function parseTokensState(state: string[]): { text: string; tokens: number[]; } {
+		let text: string[] = [];
+		let tokens: number[] = [];
+		for (let i = 0; i < state.length; i++) {
+			const line = state[i];
+
+			let startOffset = 0;
+			let lineText = '';
+			while (true) {
+				const firstPipeOffset = line.indexOf('|', startOffset);
+				if (firstPipeOffset === -1) {
+					break;
+				}
+				const secondPipeOffset = line.indexOf('|', firstPipeOffset + 1);
+				if (secondPipeOffset === -1) {
+					break;
+				}
+				if (firstPipeOffset + 1 === secondPipeOffset) {
+					// skip ||
+					lineText += line.substring(startOffset, secondPipeOffset + 1);
+					startOffset = secondPipeOffset + 1;
+					continue;
+				}
+
+				lineText += line.substring(startOffset, firstPipeOffset);
+				const tokenStartCharacter = lineText.length;
+				const tokenLength = secondPipeOffset - firstPipeOffset - 1;
+				const metadata = (SEMANTIC_COLOR << MetadataConsts.FOREGROUND_OFFSET);
+
+				tokens.push(i, tokenStartCharacter, tokenStartCharacter + tokenLength, metadata);
+
+				lineText += line.substr(firstPipeOffset + 1, tokenLength);
+				startOffset = secondPipeOffset + 1;
+			}
+
+			lineText += line.substring(startOffset);
+
+			text.push(lineText);
+		}
+
+		return {
+			text: text.join('\n'),
+			tokens: tokens
+		};
+	}
+
+	function extractState(model: TextModel): string[] {
+		let result: string[] = [];
+		for (let lineNumber = 1; lineNumber <= model.getLineCount(); lineNumber++) {
+			const lineTokens = model.getLineTokens(lineNumber);
+			const lineContent = model.getLineContent(lineNumber);
+
+			let lineText = '';
+			for (let i = 0; i < lineTokens.getCount(); i++) {
+				const tokenStartCharacter = lineTokens.getStartOffset(i);
+				const tokenEndCharacter = lineTokens.getEndOffset(i);
+				const metadata = lineTokens.getMetadata(i);
+				const color = TokenMetadata.getForeground(metadata);
+				const tokenText = lineContent.substring(tokenStartCharacter, tokenEndCharacter);
+				if (color === SEMANTIC_COLOR) {
+					lineText += `|${tokenText}|`;
+				} else {
+					lineText += tokenText;
+				}
+			}
+
+			result.push(lineText);
+		}
+		return result;
+	}
+
+	// function extractState
+
+	function testTokensAdjustment(rawInitialState: string[], edits: IIdentifiedSingleEditOperation[], rawFinalState: string[]) {
+		const initialState = parseTokensState(rawInitialState);
+		const model = TextModel.createFromString(initialState.text);
+		model.setSemanticTokens([new MultilineTokens2(1, new SparseEncodedTokens(new Uint32Array(initialState.tokens)))]);
+
+		model.applyEdits(edits);
+
+		const actualState = extractState(model);
+		assert.deepEqual(actualState, rawFinalState);
+
+		model.dispose();
+	}
+
+	test('issue #86303 - color shifting between different tokens', () => {
+		testTokensAdjustment(
+			[
+				`import { |URI| } from 'vs/base/common/uri';`,
+				`const foo = |URI|.parse('hey');`
+			],
+			[
+				{ range: new Range(2, 9, 2, 10), text: '' }
+			],
+			[
+				`import { |URI| } from 'vs/base/common/uri';`,
+				`const fo = |URI|.parse('hey');`
+			]
+		);
+	});
+
+});
