Skip to content

Commit a19857d

Browse files
author
Benjamin Pasero
committed
scorer - introduce normalized path
1 parent d7d1147 commit a19857d

6 files changed

Lines changed: 96 additions & 53 deletions

File tree

‎src/vs/base/common/fuzzyScorer.ts‎

Lines changed: 53 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -25,15 +25,15 @@ export function score(target: string, query: IPreparedQuery, fuzzy: boolean): Sc
2525
return scoreMultiple(target, query.values, fuzzy);
2626
}
2727

28-
return scoreSingle(target, query.value, query.valueLowercase, fuzzy);
28+
return scoreSingle(target, query.normalized, query.normalizedLowercase, fuzzy);
2929
}
3030

3131
function scoreMultiple(target: string, query: IPreparedQueryPiece[], fuzzy: boolean): Score {
3232
let totalScore = NO_MATCH;
3333
const totalPositions: number[] = [];
3434

35-
for (const { value, valueLowercase } of query) {
36-
const [scoreValue, positions] = scoreSingle(target, value, valueLowercase, fuzzy);
35+
for (const { normalized, normalizedLowercase } of query) {
36+
const [scoreValue, positions] = scoreSingle(target, normalized, normalizedLowercase, fuzzy);
3737
if (scoreValue === NO_MATCH) {
3838
// if a single query value does not match, return with
3939
// no score entirely, we require all queries to match
@@ -338,11 +338,26 @@ const LABEL_CAMELCASE_SCORE = 1 << 16;
338338
const LABEL_SCORE_THRESHOLD = 1 << 15;
339339

340340
export interface IPreparedQueryPiece {
341+
342+
/**
343+
* The original query as provided as input.
344+
*/
341345
original: string;
342346
originalLowercase: string;
343347

344-
value: string;
345-
valueLowercase: string;
348+
/**
349+
* Original normalized to platform separators:
350+
* - Windows: \
351+
* - Posix: /
352+
*/
353+
pathNormalized: string;
354+
355+
/**
356+
* In addition to the normalized path, will have
357+
* whitespace and wildcards removed.
358+
*/
359+
normalized: string;
360+
normalizedLowercase: string;
346361
}
347362

348363
export interface IPreparedQuery extends IPreparedQueryPiece {
@@ -364,47 +379,58 @@ export function prepareQuery(original: string): IPreparedQuery {
364379
}
365380

366381
const originalLowercase = original.toLowerCase();
367-
const value = prepareQueryValue(original);
368-
const valueLowercase = value.toLowerCase();
369-
const containsPathSeparator = value.indexOf(sep) >= 0;
382+
const { pathNormalized, normalized, normalizedLowercase } = normalizeQuery(original);
383+
const containsPathSeparator = pathNormalized.indexOf(sep) >= 0;
370384

371385
let values: IPreparedQueryPiece[] | undefined = undefined;
372386

373387
const originalSplit = original.split(MULTIPL_QUERY_VALUES_SEPARATOR);
374388
if (originalSplit.length > 1) {
375389
for (const originalPiece of originalSplit) {
376-
const valuePiece = prepareQueryValue(originalPiece);
377-
if (valuePiece) {
390+
const {
391+
pathNormalized: pathNormalizedPiece,
392+
normalized: normalizedPiece,
393+
normalizedLowercase: normalizedLowercasePiece
394+
} = normalizeQuery(originalPiece);
395+
396+
if (normalizedPiece) {
378397
if (!values) {
379398
values = [];
380399
}
381400

382401
values.push({
383402
original: originalPiece,
384403
originalLowercase: originalPiece.toLowerCase(),
385-
value: valuePiece,
386-
valueLowercase: valuePiece.toLowerCase()
404+
pathNormalized: pathNormalizedPiece,
405+
normalized: normalizedPiece,
406+
normalizedLowercase: normalizedLowercasePiece
387407
});
388408
}
389409
}
390410
}
391411

392-
return { original, originalLowercase, value, valueLowercase, values, containsPathSeparator };
412+
return { original, originalLowercase, pathNormalized, normalized, normalizedLowercase, values, containsPathSeparator };
393413
}
394414

395-
function prepareQueryValue(original: string): string {
396-
let value = stripWildcards(original).replace(/\s/g, ''); // get rid of all wildcards and whitespace
415+
function normalizeQuery(original: string): { pathNormalized: string, normalized: string, normalizedLowercase: string } {
416+
let pathNormalized: string;
397417
if (isWindows) {
398-
value = value.replace(/\//g, sep); // Help Windows users to search for paths when using slash
418+
pathNormalized = original.replace(/\//g, sep); // Help Windows users to search for paths when using slash
399419
} else {
400-
value = value.replace(/\\/g, sep); // Help macOS/Linux users to search for paths when using backslash
420+
pathNormalized = original.replace(/\\/g, sep); // Help macOS/Linux users to search for paths when using backslash
401421
}
402422

403-
return value;
423+
const normalized = stripWildcards(pathNormalized).replace(/\s/g, '');
424+
425+
return {
426+
pathNormalized,
427+
normalized,
428+
normalizedLowercase: normalized.toLowerCase()
429+
};
404430
}
405431

406432
export function scoreItem<T>(item: T, query: IPreparedQuery, fuzzy: boolean, accessor: IItemAccessor<T>, cache: ScorerCache): IItemScore {
407-
if (!item || !query.value) {
433+
if (!item || !query.normalized) {
408434
return NO_ITEM_SCORE; // we need an item and query to score on at least
409435
}
410436

@@ -417,9 +443,9 @@ export function scoreItem<T>(item: T, query: IPreparedQuery, fuzzy: boolean, acc
417443

418444
let cacheHash: string;
419445
if (description) {
420-
cacheHash = `${label}${description}${query.value}${fuzzy}`;
446+
cacheHash = `${label}${description}${query.normalized}${fuzzy}`;
421447
} else {
422-
cacheHash = `${label}${query.value}${fuzzy}`;
448+
cacheHash = `${label}${query.normalized}${fuzzy}`;
423449
}
424450

425451
const cached = cache[cacheHash];
@@ -455,7 +481,7 @@ function createMatches(offsets: undefined | number[]): IMatch[] {
455481
function doScoreItem(label: string, description: string | undefined, path: string | undefined, query: IPreparedQuery, fuzzy: boolean): IItemScore {
456482

457483
// 1.) treat identity matches on full path highest
458-
if (path && (isLinux ? query.original === path : equalsIgnoreCase(query.original, path))) {
484+
if (path && (isLinux ? query.pathNormalized === path : equalsIgnoreCase(query.pathNormalized, path))) {
459485
return { score: PATH_IDENTITY_SCORE, labelMatch: [{ start: 0, end: label.length }], descriptionMatch: description ? [{ start: 0, end: description.length }] : undefined };
460486
}
461487

@@ -464,13 +490,13 @@ function doScoreItem(label: string, description: string | undefined, path: strin
464490
if (preferLabelMatches) {
465491

466492
// 2.) treat prefix matches on the label second highest
467-
const prefixLabelMatch = matchesPrefix(query.value, label);
493+
const prefixLabelMatch = matchesPrefix(query.normalized, label);
468494
if (prefixLabelMatch) {
469495
return { score: LABEL_PREFIX_SCORE, labelMatch: prefixLabelMatch };
470496
}
471497

472498
// 3.) treat camelcase matches on the label third highest
473-
const camelcaseLabelMatch = matchesCamelCase(query.value, label);
499+
const camelcaseLabelMatch = matchesCamelCase(query.normalized, label);
474500
if (camelcaseLabelMatch) {
475501
return { score: LABEL_CAMELCASE_SCORE, labelMatch: camelcaseLabelMatch };
476502
}
@@ -702,17 +728,17 @@ function fallbackCompare<T>(itemA: T, itemB: T, query: IPreparedQuery, accessor:
702728

703729
// compare by label
704730
if (labelA !== labelB) {
705-
return compareAnything(labelA, labelB, query.value);
731+
return compareAnything(labelA, labelB, query.normalized);
706732
}
707733

708734
// compare by description
709735
if (descriptionA && descriptionB && descriptionA !== descriptionB) {
710-
return compareAnything(descriptionA, descriptionB, query.value);
736+
return compareAnything(descriptionA, descriptionB, query.normalized);
711737
}
712738

713739
// compare by path
714740
if (pathA && pathB && pathA !== pathB) {
715-
return compareAnything(pathA, pathB, query.value);
741+
return compareAnything(pathA, pathB, query.normalized);
716742
}
717743

718744
// equal

‎src/vs/base/test/common/fuzzyScorer.test.ts‎

Lines changed: 32 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -857,41 +857,58 @@ suite('Fuzzy Scorer', () => {
857857
});
858858

859859
test('prepareQuery', () => {
860-
assert.equal(scorer.prepareQuery(' f*a ').value, 'fa');
860+
assert.equal(scorer.prepareQuery(' f*a ').normalized, 'fa');
861861
assert.equal(scorer.prepareQuery('model Tester.ts').original, 'model Tester.ts');
862862
assert.equal(scorer.prepareQuery('model Tester.ts').originalLowercase, 'model Tester.ts'.toLowerCase());
863-
assert.equal(scorer.prepareQuery('model Tester.ts').value, 'modelTester.ts');
864-
assert.equal(scorer.prepareQuery('Model Tester.ts').valueLowercase, 'modeltester.ts');
863+
assert.equal(scorer.prepareQuery('model Tester.ts').normalized, 'modelTester.ts');
864+
assert.equal(scorer.prepareQuery('Model Tester.ts').normalizedLowercase, 'modeltester.ts');
865865
assert.equal(scorer.prepareQuery('ModelTester.ts').containsPathSeparator, false);
866866
assert.equal(scorer.prepareQuery('Model' + sep + 'Tester.ts').containsPathSeparator, true);
867867

868868
// with spaces
869869
let query = scorer.prepareQuery('He*llo World');
870870
assert.equal(query.original, 'He*llo World');
871-
assert.equal(query.value, 'HelloWorld');
872-
assert.equal(query.valueLowercase, 'HelloWorld'.toLowerCase());
871+
assert.equal(query.normalized, 'HelloWorld');
872+
assert.equal(query.normalizedLowercase, 'HelloWorld'.toLowerCase());
873873
assert.equal(query.values?.length, 2);
874874
assert.equal(query.values?.[0].original, 'He*llo');
875-
assert.equal(query.values?.[0].value, 'Hello');
876-
assert.equal(query.values?.[0].valueLowercase, 'Hello'.toLowerCase());
875+
assert.equal(query.values?.[0].normalized, 'Hello');
876+
assert.equal(query.values?.[0].normalizedLowercase, 'Hello'.toLowerCase());
877877
assert.equal(query.values?.[1].original, 'World');
878-
assert.equal(query.values?.[1].value, 'World');
879-
assert.equal(query.values?.[1].valueLowercase, 'World'.toLowerCase());
878+
assert.equal(query.values?.[1].normalized, 'World');
879+
assert.equal(query.values?.[1].normalizedLowercase, 'World'.toLowerCase());
880880

881881
// with spaces that are empty
882882
query = scorer.prepareQuery(' Hello World ');
883883
assert.equal(query.original, ' Hello World ');
884884
assert.equal(query.originalLowercase, ' Hello World '.toLowerCase());
885-
assert.equal(query.value, 'HelloWorld');
886-
assert.equal(query.valueLowercase, 'HelloWorld'.toLowerCase());
885+
assert.equal(query.normalized, 'HelloWorld');
886+
assert.equal(query.normalizedLowercase, 'HelloWorld'.toLowerCase());
887887
assert.equal(query.values?.length, 2);
888888
assert.equal(query.values?.[0].original, 'Hello');
889889
assert.equal(query.values?.[0].originalLowercase, 'Hello'.toLowerCase());
890-
assert.equal(query.values?.[0].value, 'Hello');
891-
assert.equal(query.values?.[0].valueLowercase, 'Hello'.toLowerCase());
890+
assert.equal(query.values?.[0].normalized, 'Hello');
891+
assert.equal(query.values?.[0].normalizedLowercase, 'Hello'.toLowerCase());
892892
assert.equal(query.values?.[1].original, 'World');
893893
assert.equal(query.values?.[1].originalLowercase, 'World'.toLowerCase());
894-
assert.equal(query.values?.[1].value, 'World');
895-
assert.equal(query.values?.[1].valueLowercase, 'World'.toLowerCase());
894+
assert.equal(query.values?.[1].normalized, 'World');
895+
assert.equal(query.values?.[1].normalizedLowercase, 'World'.toLowerCase());
896+
897+
// Path related
898+
if (isWindows) {
899+
assert.equal(scorer.prepareQuery('C:\\some\\path').pathNormalized, 'C:\\some\\path');
900+
assert.equal(scorer.prepareQuery('C:\\some\\path').normalized, 'C:\\some\\path');
901+
assert.equal(scorer.prepareQuery('C:\\some\\path').containsPathSeparator, true);
902+
assert.equal(scorer.prepareQuery('C:/some/path').pathNormalized, 'C:\\some\\path');
903+
assert.equal(scorer.prepareQuery('C:/some/path').normalized, 'C:\\some\\path');
904+
assert.equal(scorer.prepareQuery('C:/some/path').containsPathSeparator, true);
905+
} else {
906+
assert.equal(scorer.prepareQuery('/some/path').pathNormalized, '/some/path');
907+
assert.equal(scorer.prepareQuery('/some/path').normalized, '/some/path');
908+
assert.equal(scorer.prepareQuery('/some/path').containsPathSeparator, true);
909+
assert.equal(scorer.prepareQuery('\\some\\path').pathNormalized, '/some/path');
910+
assert.equal(scorer.prepareQuery('\\some\\path').normalized, '/some/path');
911+
assert.equal(scorer.prepareQuery('\\some\\path').containsPathSeparator, true);
912+
}
896913
});
897914
});

‎src/vs/workbench/browser/parts/editor/editorQuickAccess.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ export abstract class BaseEditorQuickAccessProvider extends PickerQuickAccessPro
6262

6363
// Filtering
6464
const filteredEditorEntries = this.doGetEditorPickItems().filter(entry => {
65-
if (!query.value) {
65+
if (!query.normalized) {
6666
return true;
6767
}
6868

@@ -79,7 +79,7 @@ export abstract class BaseEditorQuickAccessProvider extends PickerQuickAccessPro
7979
});
8080

8181
// Sorting
82-
if (query.value) {
82+
if (query.normalized) {
8383
const groups = this.editorGroupService.getGroups(GroupsOrder.GRID_APPEARANCE).map(group => group.id);
8484
filteredEditorEntries.sort((entryA, entryB) => {
8585
if (entryA.groupId !== entryB.groupId) {

‎src/vs/workbench/contrib/search/browser/anythingQuickAccess.ts‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -402,7 +402,7 @@ export class AnythingQuickAccessProvider extends PickerQuickAccessProvider<IAnyt
402402
const configuration = this.configuration;
403403

404404
// Just return all history entries if not searching
405-
if (!query.value) {
405+
if (!query.normalized) {
406406
return this.historyService.getHistory().map(editor => this.createAnythingPick(editor, configuration));
407407
}
408408

@@ -448,9 +448,9 @@ export class AnythingQuickAccessProvider extends PickerQuickAccessProvider<IAnyt
448448

449449
//#region File Search
450450

451-
private fileQueryDelayer = this._register(new ThrottledDelayer<URI[]>(AnythingQuickAccessProvider.TYPING_SEARCH_DELAY));
451+
private readonly fileQueryDelayer = this._register(new ThrottledDelayer<URI[]>(AnythingQuickAccessProvider.TYPING_SEARCH_DELAY));
452452

453-
private fileQueryBuilder = this.instantiationService.createInstance(QueryBuilder);
453+
private readonly fileQueryBuilder = this.instantiationService.createInstance(QueryBuilder);
454454

455455
private createFileQueryCache(): FileQueryCacheState {
456456
return new FileQueryCacheState(
@@ -462,7 +462,7 @@ export class AnythingQuickAccessProvider extends PickerQuickAccessProvider<IAnyt
462462
}
463463

464464
private async getFilePicks(query: IPreparedQuery, excludes: ResourceMap<boolean>, token: CancellationToken): Promise<Array<IAnythingQuickPickItem>> {
465-
if (!query.value) {
465+
if (!query.normalized) {
466466
return [];
467467
}
468468

@@ -692,9 +692,9 @@ export class AnythingQuickAccessProvider extends PickerQuickAccessProvider<IAnyt
692692
private async getWorkspaceSymbolPicks(query: IPreparedQuery, token: CancellationToken): Promise<Array<IAnythingQuickPickItem>> {
693693
const configuration = this.configuration;
694694
if (
695-
!query.value || // we need a value for search for
696-
!configuration.includeSymbols || // we need to enable symbols in search
697-
this.pickState.lastRange // a range is an indicator for just searching for files
695+
!query.normalized || // we need a value for search for
696+
!configuration.includeSymbols || // we need to enable symbols in search
697+
this.pickState.lastRange // a range is an indicator for just searching for files
698698
) {
699699
return [];
700700
}

‎src/vs/workbench/services/search/node/fileSearch.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,7 @@ export class FileWalker {
7777
this.errors = [];
7878

7979
if (this.filePattern) {
80-
this.normalizedFilePatternLowercase = prepareQuery(this.filePattern).valueLowercase;
80+
this.normalizedFilePatternLowercase = prepareQuery(this.filePattern).normalizedLowercase;
8181
}
8282

8383
this.globalExcludePattern = config.excludePattern && glob.parse(config.excludePattern);

‎src/vs/workbench/services/search/node/rawSearchService.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -312,7 +312,7 @@ export class SearchService implements IRawSearchService {
312312

313313
// Pattern match on results
314314
const results: IRawFileMatch[] = [];
315-
const normalizedSearchValueLowercase = prepareQuery(searchValue).valueLowercase;
315+
const normalizedSearchValueLowercase = prepareQuery(searchValue).normalizedLowercase;
316316
for (const entry of cachedEntries) {
317317

318318
// Check if this entry is a match for the search value

0 commit comments

Comments
 (0)