Skip to content

Commit 100adeb

Browse files
mjbvzmjbvz
committed
Use more generic word pattern for markdown
Try to better support handle of unicode in markdown by using a word definition that use unicode character class Fixes microsoft#15177
1 parent 0795547 commit 100adeb

6 files changed

Lines changed: 26 additions & 7 deletions

File tree

‎extensions/markdown-language-features/src/extension.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,9 @@ export function activate(context: vscode.ExtensionContext) {
4040
const previewManager = new MarkdownPreviewManager(contentProvider, logger, contributions);
4141
context.subscriptions.push(previewManager);
4242

43+
context.subscriptions.push(vscode.languages.setLanguageConfiguration('markdown', {
44+
wordPattern: /(\p{Alphabetic}|\p{Number})+/ug
45+
}));
4346
context.subscriptions.push(vscode.languages.registerDocumentSymbolProvider(selector, symbolProvider));
4447
context.subscriptions.push(vscode.languages.registerDocumentLinkProvider(selector, new LinkProvider()));
4548
context.subscriptions.push(vscode.languages.registerFoldingRangeProvider(selector, new MarkdownFoldingProvider(engine)));

‎src/vs/base/common/marshalling.ts‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
*--------------------------------------------------------------------------------------------*/
55

66
import { URI } from 'vs/base/common/uri';
7+
import { regExpFlags } from 'vs/base/common/strings';
78

89
export function stringify(obj: any): string {
910
return JSON.stringify(obj, replacer);
@@ -24,8 +25,8 @@ function replacer(key: string, value: any): any {
2425
if (value instanceof RegExp) {
2526
return {
2627
$mid: 2,
27-
source: (<RegExp>value).source,
28-
flags: ((<RegExp>value).global ? 'g' : '') + ((<RegExp>value).ignoreCase ? 'i' : '') + ((<RegExp>value).multiline ? 'm' : ''),
28+
source: value.source,
29+
flags: regExpFlags(value),
2930
};
3031
}
3132
return value;

‎src/vs/base/common/strings.ts‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -188,6 +188,7 @@ export interface RegExpOptions {
188188
wholeWord?: boolean;
189189
multiline?: boolean;
190190
global?: boolean;
191+
unicode?: boolean;
191192
}
192193

193194
export function createRegExp(searchString: string, isRegex: boolean, options: RegExpOptions = {}): RegExp {
@@ -215,6 +216,9 @@ export function createRegExp(searchString: string, isRegex: boolean, options: Re
215216
if (options.multiline) {
216217
modifiers += 'm';
217218
}
219+
if (options.unicode) {
220+
modifiers += 'u';
221+
}
218222

219223
return new RegExp(searchString, modifiers);
220224
}
@@ -236,6 +240,13 @@ export function regExpContainsBackreference(regexpValue: string): boolean {
236240
return !!regexpValue.match(/([^\\]|^)(\\\\)*\\\d+/);
237241
}
238242

243+
export function regExpFlags(regexp: RegExp): string {
244+
return (regexp.global ? 'g' : '')
245+
+ (regexp.ignoreCase ? 'i' : '')
246+
+ (regexp.multiline ? 'm' : '')
247+
+ ((regexp as any).unicode ? 'u' : '');
248+
}
249+
239250
/**
240251
* Returns first index of the string that is not whitespace.
241252
* If string is empty or contains only whitespaces, returns -1

‎src/vs/editor/common/model/wordHelper.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,9 @@ export function ensureValidWordDefinition(wordDefinition?: RegExp | null): RegEx
4141
if (wordDefinition.multiline) {
4242
flags += 'm';
4343
}
44+
if ((wordDefinition as any).unicode) {
45+
flags += 'u';
46+
}
4447
result = new RegExp(wordDefinition.source, flags);
4548
} else {
4649
result = wordDefinition;

‎src/vs/editor/common/services/editorWorkerServiceImpl.ts‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ import { EditorSimpleWorkerImpl } from 'vs/editor/common/services/editorSimpleWo
1919
import { IDiffComputationResult, IEditorWorkerService } from 'vs/editor/common/services/editorWorkerService';
2020
import { IModelService } from 'vs/editor/common/services/modelService';
2121
import { ITextResourceConfigurationService } from 'vs/editor/common/services/resourceConfiguration';
22+
import { regExpFlags } from 'vs/base/common/strings';
2223

2324
/**
2425
* Stop syncing a model to the worker if it was not needed for 1 min.
@@ -413,7 +414,7 @@ export class EditorWorkerClient extends Disposable {
413414
}
414415
let wordDefRegExp = LanguageConfigurationRegistry.getWordDefinition(model.getLanguageIdentifier().id);
415416
let wordDef = wordDefRegExp.source;
416-
let wordDefFlags = (wordDefRegExp.global ? 'g' : '') + (wordDefRegExp.ignoreCase ? 'i' : '') + (wordDefRegExp.multiline ? 'm' : '');
417+
let wordDefFlags = regExpFlags(wordDefRegExp);
417418
return proxy.textualSuggest(resource.toString(), position, wordDef, wordDefFlags);
418419
});
419420
}
@@ -426,7 +427,7 @@ export class EditorWorkerClient extends Disposable {
426427
}
427428
let wordDefRegExp = LanguageConfigurationRegistry.getWordDefinition(model.getLanguageIdentifier().id);
428429
let wordDef = wordDefRegExp.source;
429-
let wordDefFlags = (wordDefRegExp.global ? 'g' : '') + (wordDefRegExp.ignoreCase ? 'i' : '') + (wordDefRegExp.multiline ? 'm' : '');
430+
let wordDefFlags = regExpFlags(wordDefRegExp);
430431
return proxy.computeWordRanges(resource.toString(), range, wordDef, wordDefFlags);
431432
});
432433
}
@@ -439,7 +440,7 @@ export class EditorWorkerClient extends Disposable {
439440
}
440441
let wordDefRegExp = LanguageConfigurationRegistry.getWordDefinition(model.getLanguageIdentifier().id);
441442
let wordDef = wordDefRegExp.source;
442-
let wordDefFlags = (wordDefRegExp.global ? 'g' : '') + (wordDefRegExp.ignoreCase ? 'i' : '') + (wordDefRegExp.multiline ? 'm' : '');
443+
let wordDefFlags = regExpFlags(wordDefRegExp);
443444
return proxy.navigateValueSet(resource.toString(), range, up, wordDef, wordDefFlags);
444445
});
445446
}

‎src/vs/workbench/api/node/extHostLanguageFeatures.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ import { ExtHostCommands, CommandsConverter } from 'vs/workbench/api/node/extHos
1616
import { ExtHostDiagnostics } from 'vs/workbench/api/node/extHostDiagnostics';
1717
import { asThenable } from 'vs/base/common/async';
1818
import { MainContext, MainThreadLanguageFeaturesShape, ExtHostLanguageFeaturesShape, ObjectIdentifier, IRawColorInfo, IMainContext, IdObject, ISerializedRegExp, ISerializedIndentationRule, ISerializedOnEnterRule, ISerializedLanguageConfiguration, WorkspaceSymbolDto, SuggestResultDto, WorkspaceSymbolsDto, SuggestionDto, CodeActionDto, ISerializedDocumentFilter, WorkspaceEditDto, ISerializedSignatureHelpProviderMetadata } from './extHost.protocol';
19-
import { regExpLeadsToEndlessLoop } from 'vs/base/common/strings';
19+
import { regExpLeadsToEndlessLoop, regExpFlags } from 'vs/base/common/strings';
2020
import { IPosition } from 'vs/editor/common/core/position';
2121
import { IRange, Range as EditorRange } from 'vs/editor/common/core/range';
2222
import { isFalsyOrEmpty, isNonEmptyArray } from 'vs/base/common/arrays';
@@ -1299,7 +1299,7 @@ export class ExtHostLanguageFeatures implements ExtHostLanguageFeaturesShape {
12991299
}
13001300
return {
13011301
pattern: regExp.source,
1302-
flags: (regExp.global ? 'g' : '') + (regExp.ignoreCase ? 'i' : '') + (regExp.multiline ? 'm' : ''),
1302+
flags: regExpFlags(regExp),
13031303
};
13041304
}
13051305

0 commit comments

Comments
 (0)