Skip to content

Commit 70d2508

Browse files
alexdimaalexdima
committed
Fix regression in testDecodeTextMateToken (microsoft#4828)
1 parent b6fdf09 commit 70d2508

2 files changed

Lines changed: 115 additions & 49 deletions

File tree

‎src/vs/editor/node/textMate/TMSyntax.ts‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -157,7 +157,7 @@ export class DecodeMap {
157157
if (!tokenId) {
158158
tokenId = (++this.lastAssignedId);
159159
this.tokenToTokenId[token] = tokenId;
160-
this.tokenIdToToken[tokenId] = token + ' ';
160+
this.tokenIdToToken[tokenId] = token;
161161
}
162162
tokens.push(tokenId);
163163
}
@@ -168,9 +168,16 @@ export class DecodeMap {
168168

169169
public getToken(tokenMap:boolean[]): string {
170170
let result = '';
171-
for (let i = 1; i < this.lastAssignedId; i++) {
171+
let isFirst = true;
172+
for (let i = 1; i <= this.lastAssignedId; i++) {
172173
if (tokenMap[i]) {
173-
result += this.tokenIdToToken[i];
174+
if (isFirst) {
175+
isFirst = false;
176+
result += this.tokenIdToToken[i];
177+
} else {
178+
result += '.';
179+
result += this.tokenIdToToken[i];
180+
}
174181
}
175182
}
176183
return result;

‎src/vs/editor/test/node/textMate/TMSyntax.test.ts‎

Lines changed: 105 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -9,59 +9,118 @@ import {decodeTextMateToken, DecodeMap} from 'vs/editor/node/textMate/TMSyntax';
99

1010
suite('textMate', () => {
1111

12-
test('decodeTextMateToken', () => {
12+
function assertRelaxedEqual(a:string, b:string): void {
13+
let relaxString = (str:string) => {
14+
let pieces = str.split('.');
15+
pieces.sort();
16+
return pieces.join('.');
17+
};
18+
assert.equal(relaxString(a), relaxString(b));
19+
}
20+
21+
function slowDecodeTextMateToken(scopes:string[]): string {
22+
let allTokensMap: {[token:string]:boolean;} = Object.create(null);
23+
for (let i = 1; i < scopes.length; i++) {
24+
let pieces =scopes[i].split('.');
25+
for (let j = 0; j < pieces.length; j++) {
26+
allTokensMap[pieces[j]] = true;
27+
}
28+
}
29+
return Object.keys(allTokensMap).join('.');
30+
}
31+
32+
function testOneDecodeTextMateToken(decodeMap:DecodeMap, scopes:string[], expected:string): void {
33+
let actual = decodeTextMateToken(decodeMap, scopes);
34+
assert.equal(actual, expected);
35+
36+
// Sanity-check
37+
let alternativeExpected = slowDecodeTextMateToken(scopes);
38+
assertRelaxedEqual(actual, alternativeExpected);
39+
}
40+
41+
function testDecodeTextMateToken(input:string[][], expected:string[]): void {
1342
let decodeMap = new DecodeMap();
1443

15-
let input = getTestScopes();
16-
let actual:string[] = [];
17-
for (let i = 0, len = input.length; i < len; i++) {
18-
actual.push(decodeTextMateToken(decodeMap, input[i]));
44+
for (let i = 0; i < input.length; i++) {
45+
testOneDecodeTextMateToken(decodeMap, input[i], expected[i]);
1946
}
47+
}
48+
49+
test('decodeTextMateToken JSON regression', () => {
50+
let input = [
51+
['source.json','meta.structure.dictionary.json'],
52+
['source.json','meta.structure.dictionary.json','support.type.property-name.json','punctuation.support.type.property-name.begin.json'],
53+
['source.json','meta.structure.dictionary.json','support.type.property-name.json'],
54+
['source.json','meta.structure.dictionary.json','support.type.property-name.json','punctuation.support.type.property-name.end.json'],
55+
['source.json','meta.structure.dictionary.json','meta.structure.dictionary.value.json','punctuation.separator.dictionary.key-value.json'],
56+
['source.json','meta.structure.dictionary.json','meta.structure.dictionary.value.json'],
57+
['source.json','meta.structure.dictionary.json','meta.structure.dictionary.value.json','string.quoted.double.json','punctuation.definition.string.begin.json'],
58+
['source.json','meta.structure.dictionary.json','meta.structure.dictionary.value.json','string.quoted.double.json','punctuation.definition.string.end.json'],
59+
['source.json','meta.structure.dictionary.json','meta.structure.dictionary.value.json','punctuation.separator.dictionary.pair.json']
60+
];
61+
62+
let expected = [
63+
'meta.structure.dictionary.json',
64+
'meta.structure.dictionary.json.support.type.property-name.punctuation.begin',
65+
'meta.structure.dictionary.json.support.type.property-name',
66+
'meta.structure.dictionary.json.support.type.property-name.punctuation.end',
67+
'meta.structure.dictionary.json.punctuation.value.separator.key-value',
68+
'meta.structure.dictionary.json.value',
69+
'meta.structure.dictionary.json.punctuation.begin.value.string.quoted.double.definition',
70+
'meta.structure.dictionary.json.punctuation.end.value.string.quoted.double.definition',
71+
'meta.structure.dictionary.json.punctuation.value.separator.pair'
72+
];
73+
74+
testDecodeTextMateToken(input, expected);
75+
});
76+
77+
test('decodeTextMateToken', () => {
78+
let input = getTestScopes();
2079

2180
let expected = [
22-
'meta function js decl block type parameters paren cover object method declaration field member entity ',
23-
'meta function js decl block type parameters paren cover object method declaration field member parameter brace ',
24-
'meta function js decl block type parameters paren cover object method declaration field member parameter ',
25-
'meta function js decl block type parameters paren cover object method declaration field member name parameter ',
26-
'meta function js decl block type parameters paren cover object method declaration field member parameter ',
27-
'meta function js decl block type parameters paren cover object method declaration field member name parameter ',
28-
'meta function js decl block type parameters paren cover object method declaration field member parameter ',
29-
'meta function js decl block type parameters paren cover object method declaration field member name parameter ',
30-
'meta function js decl block type parameters paren cover object method declaration field member parameter brace round ',
31-
'meta function js decl block type parameters paren cover object method declaration field member ',
32-
'meta function js decl block type parameters paren cover object method declaration field member entity name ',
33-
'meta function js decl block type parameters paren cover object method declaration field member parameter brace round ',
34-
'meta function js decl block type parameters paren cover object method declaration field member name parameter variable ',
35-
'meta function js decl block type parameters paren cover object method declaration field member parameter brace round ',
36-
'meta function js decl block type parameters paren cover object method declaration field member ',
37-
'meta function js decl block type parameters paren cover object method declaration field member brace ',
38-
'meta function js decl block type parameters paren cover object method declaration field member ',
39-
'meta function js decl block type parameters paren cover object method declaration field member keyword operator ',
40-
'meta function js decl block type parameters paren cover object method declaration field member string ',
41-
'meta function js decl block type parameters paren cover object method declaration field member string ',
42-
'meta function js decl block type parameters paren cover object method declaration field member string ',
43-
'meta function js decl block type parameters paren cover object method declaration field member keyword operator ',
44-
'meta function js decl block type parameters paren cover object method declaration field member ',
45-
'meta function js decl block type parameters paren cover object method declaration field member keyword operator ',
46-
'meta function js decl block type parameters paren cover object method declaration field member string double ',
47-
'meta function js decl block type parameters paren cover object method declaration field member string double ',
48-
'meta function js decl block type parameters paren cover object method declaration field member string double ',
49-
'meta function js decl block type parameters paren cover object method declaration field member ',
50-
'meta function js decl block type parameters paren cover object method declaration field member brace array literal ',
51-
'meta function js decl block type parameters paren cover object method declaration field member array literal ',
52-
'meta function js decl block type parameters paren cover object method declaration field member brace array literal ',
53-
'meta function js decl block type parameters paren cover object method declaration field member keyword operator comparison ',
54-
'meta function js decl block type parameters paren cover object method declaration field member ',
55-
'meta function js decl block type parameters paren cover object method declaration field member brace curly ',
56-
'meta function js decl block type parameters paren cover object method declaration field member brace curly ',
57-
'meta function js decl block type parameters paren cover object method declaration field member name ',
58-
'meta function js decl block type parameters paren cover object method declaration field member ',
59-
'meta function js decl block type parameters paren cover object method declaration field member name ',
60-
'meta function js decl block type parameters paren cover object method declaration field member ',
61-
'meta function js decl block type parameters paren cover object method declaration field member '
81+
'meta.function.js.decl.block.type.parameters.paren.cover.object.method.declaration.field.member.entity.name',
82+
'meta.function.js.decl.block.type.parameters.paren.cover.object.method.declaration.field.member.parameter.brace.round',
83+
'meta.function.js.decl.block.type.parameters.paren.cover.object.method.declaration.field.member.parameter',
84+
'meta.function.js.decl.block.type.parameters.paren.cover.object.method.declaration.field.member.name.parameter.variable',
85+
'meta.function.js.decl.block.type.parameters.paren.cover.object.method.declaration.field.member.parameter',
86+
'meta.function.js.decl.block.type.parameters.paren.cover.object.method.declaration.field.member.name.parameter.variable',
87+
'meta.function.js.decl.block.type.parameters.paren.cover.object.method.declaration.field.member.parameter',
88+
'meta.function.js.decl.block.type.parameters.paren.cover.object.method.declaration.field.member.name.parameter.variable',
89+
'meta.function.js.decl.block.type.parameters.paren.cover.object.method.declaration.field.member.parameter.brace.round',
90+
'meta.function.js.decl.block.type.parameters.paren.cover.object.method.declaration.field.member',
91+
'meta.function.js.decl.block.type.parameters.paren.cover.object.method.declaration.field.member.entity.name.overload',
92+
'meta.function.js.decl.block.type.parameters.paren.cover.object.method.declaration.field.member.parameter.brace.round',
93+
'meta.function.js.decl.block.type.parameters.paren.cover.object.method.declaration.field.member.name.parameter.variable',
94+
'meta.function.js.decl.block.type.parameters.paren.cover.object.method.declaration.field.member.parameter.brace.round',
95+
'meta.function.js.decl.block.type.parameters.paren.cover.object.method.declaration.field.member',
96+
'meta.function.js.decl.block.type.parameters.paren.cover.object.method.declaration.field.member.brace.curly',
97+
'meta.function.js.decl.block.type.parameters.paren.cover.object.method.declaration.field.member',
98+
'meta.function.js.decl.block.type.parameters.paren.cover.object.method.declaration.field.member.keyword.operator.comparison',
99+
'meta.function.js.decl.block.type.parameters.paren.cover.object.method.declaration.field.member.string.double',
100+
'meta.function.js.decl.block.type.parameters.paren.cover.object.method.declaration.field.member.string.double',
101+
'meta.function.js.decl.block.type.parameters.paren.cover.object.method.declaration.field.member.string.double',
102+
'meta.function.js.decl.block.type.parameters.paren.cover.object.method.declaration.field.member.keyword.operator.arithmetic',
103+
'meta.function.js.decl.block.type.parameters.paren.cover.object.method.declaration.field.member',
104+
'meta.function.js.decl.block.type.parameters.paren.cover.object.method.declaration.field.member.keyword.operator.arithmetic',
105+
'meta.function.js.decl.block.type.parameters.paren.cover.object.method.declaration.field.member.string.double',
106+
'meta.function.js.decl.block.type.parameters.paren.cover.object.method.declaration.field.member.string.double',
107+
'meta.function.js.decl.block.type.parameters.paren.cover.object.method.declaration.field.member.string.double',
108+
'meta.function.js.decl.block.type.parameters.paren.cover.object.method.declaration.field.member',
109+
'meta.function.js.decl.block.type.parameters.paren.cover.object.method.declaration.field.member.brace.array.literal.square',
110+
'meta.function.js.decl.block.type.parameters.paren.cover.object.method.declaration.field.member.array.literal',
111+
'meta.function.js.decl.block.type.parameters.paren.cover.object.method.declaration.field.member.brace.array.literal.square',
112+
'meta.function.js.decl.block.type.parameters.paren.cover.object.method.declaration.field.member.keyword.operator.comparison',
113+
'meta.function.js.decl.block.type.parameters.paren.cover.object.method.declaration.field.member',
114+
'meta.function.js.decl.block.type.parameters.paren.cover.object.method.declaration.field.member.brace.curly',
115+
'meta.function.js.decl.block.type.parameters.paren.cover.object.method.declaration.field.member.brace.curly',
116+
'meta.function.js.decl.block.type.parameters.paren.cover.object.method.declaration.field.member.name',
117+
'meta.function.js.decl.block.type.parameters.paren.cover.object.method.declaration.field.member',
118+
'meta.function.js.decl.block.type.parameters.paren.cover.object.method.declaration.field.member.name',
119+
'meta.function.js.decl.block.type.parameters.paren.cover.object.method.declaration.field.member',
120+
'meta.function.js.decl.block.type.parameters.paren.cover.object.method.declaration.field.member'
62121
];
63122

64-
assert.deepEqual(actual, expected);
123+
testDecodeTextMateToken(input, expected);
65124
});
66125
});
67126

0 commit comments

Comments
 (0)