Skip to content

Commit be624aa

Browse files
alexdimaalexdima
committed
Cover case where the partial range is contained inside a piece
1 parent 02e728e commit be624aa

2 files changed

Lines changed: 124 additions & 12 deletions

File tree

‎src/vs/editor/common/model/tokensStore.ts‎

Lines changed: 97 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -140,6 +140,14 @@ export class SparseEncodedTokens {
140140
this._tokenCount = tokens.length / 4;
141141
}
142142

143+
public toString(startLineNumber: number): string {
144+
let pieces: string[] = [];
145+
for (let i = 0; i < this._tokenCount; i++) {
146+
pieces.push(`(${this._getDeltaLine(i) + startLineNumber},${this._getStartCharacter(i)}-${this._getEndCharacter(i)})`);
147+
}
148+
return `[${pieces.join(',')}]`;
149+
}
150+
143151
public getMaxDeltaLine(): number {
144152
const tokenCount = this._getTokenCount();
145153
if (tokenCount === 0) {
@@ -229,8 +237,8 @@ export class SparseEncodedTokens {
229237
const tokenMetadata = tokens[srcOffset + 3];
230238

231239
if (
232-
(tokenDeltaLine > startDeltaLine || (tokenDeltaLine === startDeltaLine && tokenStartCharacter >= startChar))
233-
&& (tokenDeltaLine < endDeltaLine || (tokenDeltaLine === endDeltaLine && tokenEndCharacter <= endChar))
240+
(tokenDeltaLine > startDeltaLine || (tokenDeltaLine === startDeltaLine && tokenEndCharacter >= startChar))
241+
&& (tokenDeltaLine < endDeltaLine || (tokenDeltaLine === endDeltaLine && tokenStartCharacter <= endChar))
234242
) {
235243
hasDeletedTokens = true;
236244
} else {
@@ -254,6 +262,45 @@ export class SparseEncodedTokens {
254262
return firstDeltaLine;
255263
}
256264

265+
public split(startDeltaLine: number, startChar: number, endDeltaLine: number, endChar: number): [SparseEncodedTokens, SparseEncodedTokens, number] {
266+
const tokens = this._tokens;
267+
const tokenCount = this._tokenCount;
268+
let aTokens: number[] = [];
269+
let bTokens: number[] = [];
270+
let destTokens: number[] = aTokens;
271+
let destOffset = 0;
272+
let destFirstDeltaLine: number = 0;
273+
for (let i = 0; i < tokenCount; i++) {
274+
const srcOffset = 4 * i;
275+
const tokenDeltaLine = tokens[srcOffset];
276+
const tokenStartCharacter = tokens[srcOffset + 1];
277+
const tokenEndCharacter = tokens[srcOffset + 2];
278+
const tokenMetadata = tokens[srcOffset + 3];
279+
280+
if ((tokenDeltaLine > startDeltaLine || (tokenDeltaLine === startDeltaLine && tokenEndCharacter >= startChar))) {
281+
if ((tokenDeltaLine < endDeltaLine || (tokenDeltaLine === endDeltaLine && tokenStartCharacter <= endChar))) {
282+
// this token is touching the range
283+
continue;
284+
} else {
285+
// this token is after the range
286+
if (destTokens !== bTokens) {
287+
// this token is the first token after the range
288+
destTokens = bTokens;
289+
destOffset = 0;
290+
destFirstDeltaLine = tokenDeltaLine;
291+
}
292+
}
293+
}
294+
295+
destTokens[destOffset++] = tokenDeltaLine - destFirstDeltaLine;
296+
destTokens[destOffset++] = tokenStartCharacter;
297+
destTokens[destOffset++] = tokenEndCharacter;
298+
destTokens[destOffset++] = tokenMetadata;
299+
}
300+
301+
return [new SparseEncodedTokens(new Uint32Array(aTokens)), new SparseEncodedTokens(new Uint32Array(bTokens)), destFirstDeltaLine];
302+
}
303+
257304
public acceptDeleteRange(horizontalShiftForFirstLineTokens: number, startDeltaLine: number, startCharacter: number, endDeltaLine: number, endCharacter: number): void {
258305
// This is a bit complex, here are the cases I used to think about this:
259306
//
@@ -518,6 +565,10 @@ export class MultilineTokens2 {
518565
this.endLineNumber = this.startLineNumber + this.tokens.getMaxDeltaLine();
519566
}
520567

568+
public toString(): string {
569+
return this.tokens.toString(this.startLineNumber);
570+
}
571+
521572
private _updateEndLineNumber(): void {
522573
this.endLineNumber = this.startLineNumber + this.tokens.getMaxDeltaLine();
523574
}
@@ -549,6 +600,17 @@ export class MultilineTokens2 {
549600
this._updateEndLineNumber();
550601
}
551602

603+
public split(range: Range): [MultilineTokens2, MultilineTokens2] {
604+
// split tokens to two:
605+
// a) all the tokens before `range`
606+
// b) all the tokens after `range`
607+
const startLineIndex = range.startLineNumber - this.startLineNumber;
608+
const endLineIndex = range.endLineNumber - this.startLineNumber;
609+
610+
const [a, b, bDeltaLine] = this.tokens.split(startLineIndex, range.startColumn - 1, endLineIndex, range.endColumn - 1);
611+
return [new MultilineTokens2(this.startLineNumber, a), new MultilineTokens2(this.startLineNumber + bDeltaLine, b)];
612+
}
613+
552614
public applyEdit(range: IRange, text: string): void {
553615
const [eolCount, firstLineLength, lastLineLength] = countEOL(text);
554616
this.acceptEdit(range, eolCount, firstLineLength, lastLineLength, text.length > 0 ? text.charCodeAt(0) : CharCode.Null);
@@ -837,32 +899,59 @@ export class TokensStore2 {
837899
return _range;
838900
}
839901
const range = _range.plusRange(_firstRange).plusRange(_lastRange);
840-
let insertIndex = this._pieces.length;
902+
let insertPosition: { index: number; } | null = null;
841903
for (let i = 0, len = this._pieces.length; i < len; i++) {
842904
const piece = this._pieces[i];
843905
if (piece.endLineNumber < range.startLineNumber) {
906+
// this piece is before the range
844907
continue;
845908
}
909+
846910
if (piece.startLineNumber > range.endLineNumber) {
847-
insertIndex = Math.min(i, insertIndex);
911+
// this piece is after the range, so mark the spot before this piece
912+
// as a good insertion position and stop looping
913+
insertPosition = insertPosition || { index: i };
848914
break;
849915
}
916+
917+
// this piece might intersect with the range
850918
piece.removeTokens(range);
851919

852920
if (piece.isEmpty()) {
921+
// remove the piece if it became empty
853922
this._pieces.splice(i, 1);
854923
i--;
855924
len--;
856-
insertIndex--;
857925
continue;
858926
}
859927

860-
if (piece.startLineNumber >= range.endLineNumber) {
861-
insertIndex = Math.min(i, insertIndex);
928+
if (piece.endLineNumber < range.startLineNumber) {
929+
// after removal, this piece is before the range
930+
continue;
931+
}
932+
933+
if (piece.startLineNumber > range.endLineNumber) {
934+
// after removal, this piece is after the range
935+
insertPosition = insertPosition || { index: i };
936+
continue;
862937
}
938+
939+
// after removal, this piece contains the range
940+
const [a, b] = piece.split(range);
941+
this._pieces.splice(i, 1, a, b);
942+
i++;
943+
len++;
944+
945+
insertPosition = insertPosition || { index: i };
863946
}
864947

865-
this._pieces = arrays.arrayInsert(this._pieces, insertIndex, pieces);
948+
insertPosition = insertPosition || { index: this._pieces.length };
949+
950+
this._pieces = arrays.arrayInsert(this._pieces, insertPosition.index, pieces);
951+
952+
// console.log(`I HAVE ${this._pieces.length} pieces`);
953+
// console.log(`${this._pieces.map(p => p.toString()).join(', ')}`);
954+
866955
return range;
867956
}
868957

‎src/vs/editor/test/common/model/tokensStore.test.ts‎

Lines changed: 27 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -296,8 +296,31 @@ suite('TokensStore', () => {
296296
assert.equal(lineTokens.getCount(), 3);
297297
});
298298

299-
// tokensStore.ts:878 ==> INSIDE PARTIAL SET: PIECES: [(5,5-10)]
300-
// tokensStore.ts:894 ==> AFTER PARTIAL SET: PIECES: [(10,5-10),(15,5-10),(20,5-10),(25,5-10),(30,5-10),(35,5-10)], [(5,5-10)]
301-
// tokensStore.ts:878 ==> INSIDE PARTIAL SET: PIECES: [(10,5-10),(15,5-10)], [(5,5-10)]
302-
// tokensStore.ts:894 ==> AFTER PARTIAL SET: PIECES: [(20,5-10),(25,5-10),(30,5-10),(35,5-10),(40,5-10)], [(10,5-10),(15,5-10)], [(5,5-10)]
299+
test('partial tokens 3', () => {
300+
const store = new TokensStore2();
301+
302+
// setPartial: [1,1 -> 31,2], [(5,5-10),(10,5-10),(15,5-10),(20,5-10),(25,5-10),(30,5-10)]
303+
store.setPartial(new Range(1, 1, 31, 2), [
304+
new MultilineTokens2(5, new SparseEncodedTokens(new Uint32Array([
305+
0, 5, 10, 1,
306+
5, 5, 10, 2,
307+
10, 5, 10, 3,
308+
15, 5, 10, 4,
309+
20, 5, 10, 5,
310+
25, 5, 10, 6,
311+
])))
312+
]);
313+
314+
// setPartial: [11,1 -> 16,2], [(15,5-10),(20,5-10)]
315+
store.setPartial(new Range(11, 1, 16, 2), [
316+
new MultilineTokens2(10, new SparseEncodedTokens(new Uint32Array([
317+
0, 5, 10, 3,
318+
5, 5, 10, 4,
319+
])))
320+
]);
321+
322+
const lineTokens = store.addSemanticTokens(5, new LineTokens(new Uint32Array([12, 1]), `enum Enum1 {`));
323+
assert.equal(lineTokens.getCount(), 3);
324+
});
325+
303326
});

0 commit comments

Comments
 (0)