Skip to content

Commit 17f797b

Browse files
alexdimaalexdima
committed
Fixes microsoft#91936: Do not emit empty tokens in the model, add workaround for rendering empty tokens in the view
1 parent d431ec5 commit 17f797b

4 files changed

Lines changed: 147 additions & 13 deletions

File tree

‎src/vs/editor/common/model/tokensStore.ts‎

Lines changed: 17 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -781,6 +781,17 @@ export class TokensStore2 {
781781

782782
let aIndex = 0;
783783
let result: number[] = [], resultLen = 0;
784+
let lastEndOffset = 0;
785+
786+
const emitToken = (endOffset: number, metadata: number) => {
787+
if (endOffset === lastEndOffset) {
788+
return;
789+
}
790+
lastEndOffset = endOffset;
791+
result[resultLen++] = endOffset;
792+
result[resultLen++] = metadata;
793+
};
794+
784795
for (let bIndex = 0; bIndex < bLen; bIndex++) {
785796
const bStartCharacter = bTokens.getStartCharacter(bIndex);
786797
const bEndCharacter = bTokens.getEndCharacter(bIndex);
@@ -797,42 +808,36 @@ export class TokensStore2 {
797808

798809
// push any token from `a` that is before `b`
799810
while (aIndex < aLen && aTokens.getEndOffset(aIndex) <= bStartCharacter) {
800-
result[resultLen++] = aTokens.getEndOffset(aIndex);
801-
result[resultLen++] = aTokens.getMetadata(aIndex);
811+
emitToken(aTokens.getEndOffset(aIndex), aTokens.getMetadata(aIndex));
802812
aIndex++;
803813
}
804814

805815
// push the token from `a` if it intersects the token from `b`
806816
if (aIndex < aLen && aTokens.getStartOffset(aIndex) < bStartCharacter) {
807-
result[resultLen++] = bStartCharacter;
808-
result[resultLen++] = aTokens.getMetadata(aIndex);
817+
emitToken(bStartCharacter, aTokens.getMetadata(aIndex));
809818
}
810819

811820
// skip any tokens from `a` that are contained inside `b`
812821
while (aIndex < aLen && aTokens.getEndOffset(aIndex) < bEndCharacter) {
813-
result[resultLen++] = aTokens.getEndOffset(aIndex);
814-
result[resultLen++] = (aTokens.getMetadata(aIndex) & aMask) | (bMetadata & bMask);
822+
emitToken(aTokens.getEndOffset(aIndex), (aTokens.getMetadata(aIndex) & aMask) | (bMetadata & bMask));
815823
aIndex++;
816824
}
817825

818826
if (aIndex < aLen && aTokens.getEndOffset(aIndex) === bEndCharacter) {
819827
// `a` ends exactly at the same spot as `b`!
820-
result[resultLen++] = aTokens.getEndOffset(aIndex);
821-
result[resultLen++] = (aTokens.getMetadata(aIndex) & aMask) | (bMetadata & bMask);
828+
emitToken(aTokens.getEndOffset(aIndex), (aTokens.getMetadata(aIndex) & aMask) | (bMetadata & bMask));
822829
aIndex++;
823830
} else {
824831
const aMergeIndex = Math.min(Math.max(0, aIndex - 1), aLen - 1);
825832

826833
// push the token from `b`
827-
result[resultLen++] = bEndCharacter;
828-
result[resultLen++] = (aTokens.getMetadata(aMergeIndex) & aMask) | (bMetadata & bMask);
834+
emitToken(bEndCharacter, (aTokens.getMetadata(aMergeIndex) & aMask) | (bMetadata & bMask));
829835
}
830836
}
831837

832838
// push the remaining tokens from `a`
833839
while (aIndex < aLen) {
834-
result[resultLen++] = aTokens.getEndOffset(aIndex);
835-
result[resultLen++] = aTokens.getMetadata(aIndex);
840+
emitToken(aTokens.getEndOffset(aIndex), aTokens.getMetadata(aIndex));
836841
aIndex++;
837842
}
838843

‎src/vs/editor/common/viewLayout/viewLineRenderer.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -683,7 +683,7 @@ function _applyRenderWhitespace(input: RenderLineInput, lineContent: string, len
683683

684684
wasInWhitespace = isInWhitespace;
685685

686-
if (charIndex === tokenEndIndex) {
686+
while (charIndex === tokenEndIndex) {
687687
tokenIndex++;
688688
if (tokenIndex < tokensLength) {
689689
tokenType = tokens[tokenIndex].type;

‎src/vs/editor/test/common/model/tokensStore.test.ts‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -169,4 +169,47 @@ suite('TokensStore', () => {
169169
);
170170
});
171171

172+
test('issue #91936: Semantic token color highlighting fails on line with selected text', () => {
173+
const model = createTextModel(' else if ($s = 08) then \'\\b\'');
174+
model.setSemanticTokens([
175+
new MultilineTokens2(1, new SparseEncodedTokens(new Uint32Array([
176+
0, 20, 24, 245768,
177+
0, 25, 27, 245768,
178+
0, 28, 29, 16392,
179+
0, 29, 31, 262152,
180+
0, 32, 33, 16392,
181+
0, 34, 36, 98312,
182+
0, 36, 37, 16392,
183+
0, 38, 42, 245768,
184+
0, 43, 47, 180232,
185+
])))
186+
]);
187+
const lineTokens = model.getLineTokens(1);
188+
let decodedTokens: number[] = [];
189+
for (let i = 0, len = lineTokens.getCount(); i < len; i++) {
190+
decodedTokens.push(lineTokens.getEndOffset(i), lineTokens.getMetadata(i));
191+
}
192+
193+
assert.deepEqual(decodedTokens, [
194+
20, 16793600,
195+
24, 17022976,
196+
25, 16793600,
197+
27, 17022976,
198+
28, 16793600,
199+
29, 16793600,
200+
31, 17039360,
201+
32, 16793600,
202+
33, 16793600,
203+
34, 16793600,
204+
36, 16875520,
205+
37, 16793600,
206+
38, 16793600,
207+
42, 17022976,
208+
43, 16793600,
209+
47, 16957440
210+
]);
211+
212+
model.dispose();
213+
});
214+
172215
});

‎src/vs/editor/test/common/viewLayout/viewLineRenderer.test.ts‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1861,6 +1861,92 @@ suite('viewLineRenderer.renderLine 2', () => {
18611861
assert.deepEqual(actual.html, expected);
18621862
});
18631863

1864+
test('issue #91936: Semantic token color highlighting fails on line with selected text', () => {
1865+
let actual = renderViewLine(new RenderLineInput(
1866+
false,
1867+
true,
1868+
' else if ($s = 08) then \'\\b\'',
1869+
false,
1870+
true,
1871+
false,
1872+
0,
1873+
createViewLineTokens([
1874+
createPart(20, 1),
1875+
createPart(24, 15),
1876+
createPart(25, 1),
1877+
createPart(27, 15),
1878+
createPart(28, 1),
1879+
createPart(29, 1),
1880+
createPart(29, 1),
1881+
createPart(31, 16),
1882+
createPart(32, 1),
1883+
createPart(33, 1),
1884+
createPart(34, 1),
1885+
createPart(36, 6),
1886+
createPart(36, 1),
1887+
createPart(37, 1),
1888+
createPart(38, 1),
1889+
createPart(42, 15),
1890+
createPart(43, 1),
1891+
createPart(47, 11)
1892+
]),
1893+
[],
1894+
4,
1895+
0,
1896+
10,
1897+
11,
1898+
11,
1899+
10000,
1900+
'selection',
1901+
false,
1902+
false,
1903+
[new LineRange(0, 47)]
1904+
));
1905+
1906+
let expected = [
1907+
'<span>',
1908+
'<span class="mtkz" style="width:10px">\u00b7</span>',
1909+
'<span class="mtkz" style="width:10px">\u00b7</span>',
1910+
'<span class="mtkz" style="width:10px">\u00b7</span>',
1911+
'<span class="mtkz" style="width:10px">\u00b7</span>',
1912+
'<span class="mtkz" style="width:10px">\u00b7</span>',
1913+
'<span class="mtkz" style="width:10px">\u00b7</span>',
1914+
'<span class="mtkz" style="width:10px">\u00b7</span>',
1915+
'<span class="mtkz" style="width:10px">\u00b7</span>',
1916+
'<span class="mtkz" style="width:10px">\u00b7</span>',
1917+
'<span class="mtkz" style="width:10px">\u00b7</span>',
1918+
'<span class="mtkz" style="width:10px">\u00b7</span>',
1919+
'<span class="mtkz" style="width:10px">\u00b7</span>',
1920+
'<span class="mtkz" style="width:10px">\u00b7</span>',
1921+
'<span class="mtkz" style="width:10px">\u00b7</span>',
1922+
'<span class="mtkz" style="width:10px">\u00b7</span>',
1923+
'<span class="mtkz" style="width:10px">\u00b7</span>',
1924+
'<span class="mtkz" style="width:10px">\u00b7</span>',
1925+
'<span class="mtkz" style="width:10px">\u00b7</span>',
1926+
'<span class="mtkz" style="width:10px">\u00b7</span>',
1927+
'<span class="mtkz" style="width:10px">\u00b7</span>',
1928+
'<span class="mtk15">else</span>',
1929+
'<span class="mtkz" style="width:10px">\u00b7</span>',
1930+
'<span class="mtk15">if</span>',
1931+
'<span class="mtkz" style="width:10px">\u00b7</span>',
1932+
'<span class="mtk1">(</span>',
1933+
'<span class="mtk16">$s</span>',
1934+
'<span class="mtkz" style="width:10px">\u00b7</span>',
1935+
'<span class="mtk1">=</span>',
1936+
'<span class="mtkz" style="width:10px">\u00b7</span>',
1937+
'<span class="mtk6">08</span>',
1938+
'<span class="mtk1">)</span>',
1939+
'<span class="mtkz" style="width:10px">\u00b7</span>',
1940+
'<span class="mtk15">then</span>',
1941+
'<span class="mtkz" style="width:10px">\u00b7</span>',
1942+
'<span class="mtk11">\'\\b\'</span>',
1943+
'</span>'
1944+
].join('');
1945+
1946+
assert.deepEqual(actual.html, expected);
1947+
});
1948+
1949+
18641950
function createTestGetColumnOfLinePartOffset(lineContent: string, tabSize: number, parts: ViewLineToken[], expectedPartLengths: number[]): (partIndex: number, partLength: number, offset: number, expected: number) => void {
18651951
let renderLineOutput = renderViewLine(new RenderLineInput(
18661952
false,

0 commit comments

Comments
 (0)