Detect and remove unusual line terminators (fixes microsoft#96142)

alexdima · alexdima · commit 344f5c0e6746 · 2020-05-12T14:44:59.000+02:00
diff --git a/src/vs/base/common/strings.ts b/src/vs/base/common/strings.ts
@@ -728,6 +728,14 @@ export function isBasicASCII(str: string): boolean {
 	return IS_BASIC_ASCII.test(str);
 }
 
+export const UNUSUAL_LINE_TERMINATORS = /[\u2028\u2029\u0085]/; // LINE SEPARATOR (LS), PARAGRAPH SEPARATOR (PS), NEXT LINE (NEL)
+/**
+ * Returns true if `str` contains unusual line terminators, like LS, PS or NEL
+ */
+export function containsUnusualLineTerminators(str: string): boolean {
+	return UNUSUAL_LINE_TERMINATORS.test(str);
+}
+
 export function containsFullWidthCharacter(str: string): boolean {
 	for (let i = 0, len = str.length; i < len; i++) {
 		if (isFullWidthCharacter(str.charCodeAt(i))) {
diff --git a/src/vs/editor/common/config/editorOptions.ts b/src/vs/editor/common/config/editorOptions.ts
@@ -93,6 +93,11 @@ export interface IEditorOptions {
 	 * Defaults to true.
 	*/
 	renderFinalNewline?: boolean;
+	/**
+	 * Remove unusual line terminators like LINE SEPARATOR (LS), PARAGRAPH SEPARATOR (PS), NEXT LINE (NEL).
+	 * Defaults to true.
+	 */
+	removeUnusualLineTerminators?: boolean;
 	/**
 	 * Should the corresponding line be selected when clicking on the line number?
 	 * Defaults to true.
@@ -3547,6 +3552,7 @@ export const enum EditorOption {
 	quickSuggestions,
 	quickSuggestionsDelay,
 	readOnly,
+	removeUnusualLineTerminators,
 	renameOnType,
 	renderControlCharacters,
 	renderIndentGuides,
@@ -3965,6 +3971,10 @@ export const EditorOptions = {
 	readOnly: register(new EditorBooleanOption(
 		EditorOption.readOnly, 'readOnly', false,
 	)),
+	removeUnusualLineTerminators: register(new EditorBooleanOption(
+		EditorOption.removeUnusualLineTerminators, 'removeUnusualLineTerminators', true,
+		{ description: nls.localize('removeUnusualLineTerminators', "Remove unusual line terminators that might cause problems.") }
+	)),
 	renameOnType: register(new EditorBooleanOption(
 		EditorOption.renameOnType, 'renameOnType', false,
 		{ description: nls.localize('renameOnType', "Controls whether the editor auto renames on type.") }
diff --git a/src/vs/editor/common/model.ts b/src/vs/editor/common/model.ts
@@ -551,6 +551,18 @@ export interface ITextModel {
 	 */
 	mightContainRTL(): boolean;
 
+	/**
+	 * If true, the text model might contain LINE SEPARATOR (LS), PARAGRAPH SEPARATOR (PS), NEXT LINE (NEL).
+	 * If false, the text model definitely does not contain these.
+	 * @internal
+	 */
+	mightContainUnusualLineTerminators(): boolean;
+
+	/**
+	 * @internal
+	 */
+	removeUnusualLineTerminators(selections?: Selection[]): void;
+
 	/**
 	 * If true, the text model might contain non basic ASCII.
 	 * If false, the text model **contains only** basic ASCII.
@@ -1281,6 +1293,8 @@ export interface IReadonlyTextBuffer {
 	onDidChangeContent: Event<void>;
 	equals(other: ITextBuffer): boolean;
 	mightContainRTL(): boolean;
+	mightContainUnusualLineTerminators(): boolean;
+	resetMightContainUnusualLineTerminators(): void;
 	mightContainNonBasicASCII(): boolean;
 	getBOM(): string;
 	getEOL(): string;
diff --git a/src/vs/editor/common/model/pieceTreeTextBuffer/pieceTreeTextBuffer.ts b/src/vs/editor/common/model/pieceTreeTextBuffer/pieceTreeTextBuffer.ts
@@ -36,15 +36,17 @@ export class PieceTreeTextBuffer implements ITextBuffer, IDisposable {
 	private readonly _pieceTree: PieceTreeBase;
 	private readonly _BOM: string;
 	private _mightContainRTL: boolean;
+	private _mightContainUnusualLineTerminators: boolean;
 	private _mightContainNonBasicASCII: boolean;
 
 	private readonly _onDidChangeContent: Emitter<void> = new Emitter<void>();
 	public readonly onDidChangeContent: Event<void> = this._onDidChangeContent.event;
 
-	constructor(chunks: StringBuffer[], BOM: string, eol: '\r\n' | '\n', containsRTL: boolean, isBasicASCII: boolean, eolNormalized: boolean) {
+	constructor(chunks: StringBuffer[], BOM: string, eol: '\r\n' | '\n', containsRTL: boolean, containsUnusualLineTerminators: boolean, isBasicASCII: boolean, eolNormalized: boolean) {
 		this._BOM = BOM;
 		this._mightContainNonBasicASCII = !isBasicASCII;
 		this._mightContainRTL = containsRTL;
+		this._mightContainUnusualLineTerminators = containsUnusualLineTerminators;
 		this._pieceTree = new PieceTreeBase(chunks, eol, eolNormalized);
 	}
 	dispose(): void {
@@ -67,6 +69,12 @@ export class PieceTreeTextBuffer implements ITextBuffer, IDisposable {
 	public mightContainRTL(): boolean {
 		return this._mightContainRTL;
 	}
+	public mightContainUnusualLineTerminators(): boolean {
+		return this._mightContainUnusualLineTerminators;
+	}
+	public resetMightContainUnusualLineTerminators(): void {
+		this._mightContainUnusualLineTerminators = false;
+	}
 	public mightContainNonBasicASCII(): boolean {
 		return this._mightContainNonBasicASCII;
 	}
@@ -216,6 +224,7 @@ export class PieceTreeTextBuffer implements ITextBuffer, IDisposable {
 
 	public applyEdits(rawOperations: ValidAnnotatedEditOperation[], recordTrimAutoWhitespace: boolean, computeUndoEdits: boolean): ApplyEditsResult {
 		let mightContainRTL = this._mightContainRTL;
+		let mightContainUnusualLineTerminators = this._mightContainUnusualLineTerminators;
 		let mightContainNonBasicASCII = this._mightContainNonBasicASCII;
 		let canReduceOperations = true;
 
@@ -226,12 +235,20 @@ export class PieceTreeTextBuffer implements ITextBuffer, IDisposable {
 				canReduceOperations = false;
 			}
 			let validatedRange = op.range;
-			if (!mightContainRTL && op.text) {
-				// check if the new inserted text contains RTL
-				mightContainRTL = strings.containsRTL(op.text);
-			}
-			if (!mightContainNonBasicASCII && op.text) {
-				mightContainNonBasicASCII = !strings.isBasicASCII(op.text);
+			if (op.text) {
+				let textMightContainNonBasicASCII = true;
+				if (!mightContainNonBasicASCII) {
+					textMightContainNonBasicASCII = !strings.isBasicASCII(op.text);
+					mightContainNonBasicASCII = textMightContainNonBasicASCII;
+				}
+				if (!mightContainRTL && textMightContainNonBasicASCII) {
+					// check if the new inserted text contains RTL
+					mightContainRTL = strings.containsRTL(op.text);
+				}
+				if (!mightContainUnusualLineTerminators && textMightContainNonBasicASCII) {
+					// check if the new inserted text contains unusual line terminators
+					mightContainUnusualLineTerminators = strings.containsUnusualLineTerminators(op.text);
+				}
 			}
 
 			let validText = '';
@@ -340,6 +357,7 @@ export class PieceTreeTextBuffer implements ITextBuffer, IDisposable {
 
 
 		this._mightContainRTL = mightContainRTL;
+		this._mightContainUnusualLineTerminators = mightContainUnusualLineTerminators;
 		this._mightContainNonBasicASCII = mightContainNonBasicASCII;
 
 		const contentChanges = this._doApplyEdits(operations);
diff --git a/src/vs/editor/common/model/pieceTreeTextBuffer/pieceTreeTextBufferBuilder.ts b/src/vs/editor/common/model/pieceTreeTextBuffer/pieceTreeTextBufferBuilder.ts
@@ -18,6 +18,7 @@ export class PieceTreeTextBufferFactory implements ITextBufferFactory {
 		private readonly _lf: number,
 		private readonly _crlf: number,
 		private readonly _containsRTL: boolean,
+		private readonly _containsUnusualLineTerminators: boolean,
 		private readonly _isBasicASCII: boolean,
 		private readonly _normalizeEOL: boolean
 	) { }
@@ -53,7 +54,7 @@ export class PieceTreeTextBufferFactory implements ITextBufferFactory {
 			}
 		}
 
-		return new PieceTreeTextBuffer(chunks, this._bom, eol, this._containsRTL, this._isBasicASCII, this._normalizeEOL);
+		return new PieceTreeTextBuffer(chunks, this._bom, eol, this._containsRTL, this._containsUnusualLineTerminators, this._isBasicASCII, this._normalizeEOL);
 	}
 
 	public getFirstLineText(lengthLimit: number): string {
@@ -73,6 +74,7 @@ export class PieceTreeTextBufferBuilder implements ITextBufferBuilder {
 	private lf: number;
 	private crlf: number;
 	private containsRTL: boolean;
+	private containsUnusualLineTerminators: boolean;
 	private isBasicASCII: boolean;
 
 	constructor() {
@@ -87,6 +89,7 @@ export class PieceTreeTextBufferBuilder implements ITextBufferBuilder {
 		this.lf = 0;
 		this.crlf = 0;
 		this.containsRTL = false;
+		this.containsUnusualLineTerminators = false;
 		this.isBasicASCII = true;
 	}
 
@@ -140,9 +143,13 @@ export class PieceTreeTextBufferBuilder implements ITextBufferBuilder {
 			this.isBasicASCII = lineStarts.isBasicASCII;
 		}
 		if (!this.isBasicASCII && !this.containsRTL) {
-			// No need to check if is basic ASCII
+			// No need to check if it is basic ASCII
 			this.containsRTL = strings.containsRTL(chunk);
 		}
+		if (!this.isBasicASCII && !this.containsUnusualLineTerminators) {
+			// No need to check if it is basic ASCII
+			this.containsUnusualLineTerminators = strings.containsUnusualLineTerminators(chunk);
+		}
 	}
 
 	public finish(normalizeEOL: boolean = true): PieceTreeTextBufferFactory {
@@ -154,6 +161,7 @@ export class PieceTreeTextBufferBuilder implements ITextBufferBuilder {
 			this.lf,
 			this.crlf,
 			this.containsRTL,
+			this.containsUnusualLineTerminators,
 			this.isBasicASCII,
 			normalizeEOL
 		);
diff --git a/src/vs/editor/common/model/textModel.ts b/src/vs/editor/common/model/textModel.ts
@@ -37,6 +37,7 @@ import { Color } from 'vs/base/common/color';
 import { EditorTheme } from 'vs/editor/common/view/viewContext';
 import { IUndoRedoService } from 'vs/platform/undoRedo/common/undoRedo';
 import { TextChange } from 'vs/editor/common/model/textChange';
+import { Constants } from 'vs/base/common/uint';
 
 function createTextBufferBuilder() {
 	return new PieceTreeTextBufferBuilder();
@@ -699,6 +700,17 @@ export class TextModel extends Disposable implements model.ITextModel {
 		return this._buffer.mightContainRTL();
 	}
 
+	public mightContainUnusualLineTerminators(): boolean {
+		return this._buffer.mightContainUnusualLineTerminators();
+	}
+
+	public removeUnusualLineTerminators(selections: Selection[] | null = null): void {
+		const matches = this.findMatches(strings.UNUSUAL_LINE_TERMINATORS.source, false, true, false, null, false, Constants.MAX_SAFE_SMALL_INTEGER);
+		const eol = this.getEOL();
+		this._buffer.resetMightContainUnusualLineTerminators();
+		this.pushEditOperations(selections, matches.map(m => ({ range: m.range, text: eol })), () => null);
+	}
+
 	public mightContainNonBasicASCII(): boolean {
 		return this._buffer.mightContainNonBasicASCII();
 	}
@@ -1097,7 +1109,7 @@ export class TextModel extends Disposable implements model.ITextModel {
 		return this._buffer.findMatchesLineByLine(searchRange, searchData, captureMatches, limitResultCount);
 	}
 
-	public findMatches(searchString: string, rawSearchScope: any, isRegex: boolean, matchCase: boolean, wordSeparators: string, captureMatches: boolean, limitResultCount: number = LIMIT_FIND_COUNT): model.FindMatch[] {
+	public findMatches(searchString: string, rawSearchScope: any, isRegex: boolean, matchCase: boolean, wordSeparators: string | null, captureMatches: boolean, limitResultCount: number = LIMIT_FIND_COUNT): model.FindMatch[] {
 		this._assertNotDisposed();
 
 		let searchRange: Range;
diff --git a/src/vs/editor/common/standalone/standaloneEnums.ts b/src/vs/editor/common/standalone/standaloneEnums.ts
@@ -240,49 +240,50 @@ export enum EditorOption {
 	quickSuggestions = 70,
 	quickSuggestionsDelay = 71,
 	readOnly = 72,
-	renameOnType = 73,
-	renderControlCharacters = 74,
-	renderIndentGuides = 75,
-	renderFinalNewline = 76,
-	renderLineHighlight = 77,
-	renderLineHighlightOnlyWhenFocus = 78,
-	renderValidationDecorations = 79,
-	renderWhitespace = 80,
-	revealHorizontalRightPadding = 81,
-	roundedSelection = 82,
-	rulers = 83,
-	scrollbar = 84,
-	scrollBeyondLastColumn = 85,
-	scrollBeyondLastLine = 86,
-	scrollPredominantAxis = 87,
-	selectionClipboard = 88,
-	selectionHighlight = 89,
-	selectOnLineNumbers = 90,
-	showFoldingControls = 91,
-	showUnused = 92,
-	snippetSuggestions = 93,
-	smoothScrolling = 94,
-	stopRenderingLineAfter = 95,
-	suggest = 96,
-	suggestFontSize = 97,
-	suggestLineHeight = 98,
-	suggestOnTriggerCharacters = 99,
-	suggestSelection = 100,
-	tabCompletion = 101,
-	useTabStops = 102,
-	wordSeparators = 103,
-	wordWrap = 104,
-	wordWrapBreakAfterCharacters = 105,
-	wordWrapBreakBeforeCharacters = 106,
-	wordWrapColumn = 107,
-	wordWrapMinified = 108,
-	wrappingIndent = 109,
-	wrappingStrategy = 110,
-	editorClassName = 111,
-	pixelRatio = 112,
-	tabFocusMode = 113,
-	layoutInfo = 114,
-	wrappingInfo = 115
+	removeUnusualLineTerminators = 73,
+	renameOnType = 74,
+	renderControlCharacters = 75,
+	renderIndentGuides = 76,
+	renderFinalNewline = 77,
+	renderLineHighlight = 78,
+	renderLineHighlightOnlyWhenFocus = 79,
+	renderValidationDecorations = 80,
+	renderWhitespace = 81,
+	revealHorizontalRightPadding = 82,
+	roundedSelection = 83,
+	rulers = 84,
+	scrollbar = 85,
+	scrollBeyondLastColumn = 86,
+	scrollBeyondLastLine = 87,
+	scrollPredominantAxis = 88,
+	selectionClipboard = 89,
+	selectionHighlight = 90,
+	selectOnLineNumbers = 91,
+	showFoldingControls = 92,
+	showUnused = 93,
+	snippetSuggestions = 94,
+	smoothScrolling = 95,
+	stopRenderingLineAfter = 96,
+	suggest = 97,
+	suggestFontSize = 98,
+	suggestLineHeight = 99,
+	suggestOnTriggerCharacters = 100,
+	suggestSelection = 101,
+	tabCompletion = 102,
+	useTabStops = 103,
+	wordSeparators = 104,
+	wordWrap = 105,
+	wordWrapBreakAfterCharacters = 106,
+	wordWrapBreakBeforeCharacters = 107,
+	wordWrapColumn = 108,
+	wordWrapMinified = 109,
+	wrappingIndent = 110,
+	wrappingStrategy = 111,
+	editorClassName = 112,
+	pixelRatio = 113,
+	tabFocusMode = 114,
+	layoutInfo = 115,
+	wrappingInfo = 116
 }
 
 /**
diff --git a/src/vs/editor/contrib/unusualLineTerminators/unusualLineTerminators.ts b/src/vs/editor/contrib/unusualLineTerminators/unusualLineTerminators.ts
diff --git a/src/vs/editor/editor.all.ts b/src/vs/editor/editor.all.ts
diff --git a/src/vs/monaco.d.ts b/src/vs/monaco.d.ts
