Skip to content

Commit fa4b68f

Browse files
CyrusNajmabadiCyrusNajmabadi
committed
Initial test harness for incremental parser tests.
1 parent c17eb7d commit fa4b68f

6 files changed

Lines changed: 252 additions & 80 deletions

File tree

‎Jakefile‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,7 @@ var harnessSources = [
8484
].map(function (f) {
8585
return path.join(harnessDirectory, f);
8686
}).concat([
87+
"incrementalParser.ts",
8788
"services/colorization.ts",
8889
"services/documentRegistry.ts",
8990
"services/preProcessFile.ts"

‎src/compiler/parser.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -359,6 +359,7 @@ module ts {
359359
case SyntaxKind.Block:
360360
case SyntaxKind.TryBlock:
361361
case SyntaxKind.FinallyBlock:
362+
return children((<Block>node).statements);
362363
case SyntaxKind.ModuleBlock:
363364
return children((<Block>node).statements);
364365
case SyntaxKind.SourceFile:

‎src/harness/harness.ts‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,85 @@ module Utils {
111111
}
112112
});
113113
}
114+
115+
export function checkInvariants(node: ts.Node, parent: ts.Node): void {
116+
if(node) {
117+
if (node.pos < 0) {
118+
throw new Error("node.pos < 0");
119+
}
120+
if (node.end < 0) {
121+
throw new Error("node.end < 0");
122+
}
123+
if (node.end < node.pos) {
124+
throw new Error("node.end < node.pos");
125+
}
126+
if (node.parent !== parent) {
127+
throw new Error("node.parent !== parent");
128+
}
129+
if (parent) {
130+
// Make sure each child is contained within the parent.
131+
if (node.pos < parent.pos) {
132+
throw new Error("node.pos < parent.pos");
133+
}
134+
if (node.end > parent.end) {
135+
throw new Error("node.end > parent.end");
136+
}
137+
}
138+
139+
ts.forEachChild(node, child => {
140+
checkInvariants(child, node);
141+
});
142+
143+
// Make sure each of the children is in order.
144+
var currentPos = 0;
145+
ts.forEachChild(node,
146+
child => {
147+
if (child.pos < currentPos) {
148+
throw new Error("child.pos < currentPos");
149+
}
150+
currentPos = child.end;
151+
},
152+
(array: ts.NodeArray<ts.Node>) => {
153+
if (array.pos < node.pos) {
154+
throw new Error("array.pos < node.pos");
155+
}
156+
if (array.end > node.end) {
157+
throw new Error("array.end > node.end");
158+
}
159+
160+
if (array.pos < currentPos) {
161+
throw new Error("array.pos < currentPos");
162+
}
163+
for (var i = 0, n = array.length; i < n; i++) {
164+
if (array[i].pos < currentPos) {
165+
throw new Error("array[i].pos < currentPos");
166+
}
167+
currentPos = array[i].end
168+
}
169+
170+
currentPos = array.end;
171+
});
172+
173+
var childNodesAndArrays: any[] = [];
174+
ts.forEachChild(node, child => { childNodesAndArrays.push(child) }, array => { childNodesAndArrays.push(array) });
175+
176+
for (var childName in node) {
177+
if (childName === "parent" || childName === "nextContainer" || childName === "modifiers" || childName === "externalModuleIndicator") {
178+
continue;
179+
}
180+
var child = (<any>node)[childName];
181+
if (isNodeOrArray(child)) {
182+
if (childNodesAndArrays.indexOf(child) < 0) {
183+
throw new Error("Child when forEach'ing over node. " + (<any>ts).SyntaxKind[node.kind] + "-" + childName);
184+
}
185+
}
186+
}
187+
}
188+
}
189+
190+
function isNodeOrArray(a: any): boolean {
191+
return a !== undefined && typeof a.pos === "number";
192+
}
114193
}
115194

116195
module Harness.Path {

‎src/harness/test262Runner.ts‎

Lines changed: 1 addition & 76 deletions
Original file line numberDiff line numberDiff line change
@@ -21,81 +21,6 @@ class Test262BaselineRunner extends RunnerBase {
2121
return Test262BaselineRunner.basePath + "/" + filename;
2222
}
2323

24-
private static checkInvariants(node: ts.Node, parent: ts.Node): void {
25-
if (node) {
26-
if (node.pos < 0) {
27-
throw new Error("node.pos < 0");
28-
}
29-
if (node.end < 0) {
30-
throw new Error("node.end < 0");
31-
}
32-
if (node.end < node.pos) {
33-
throw new Error("node.end < node.pos");
34-
}
35-
if (node.parent !== parent) {
36-
throw new Error("node.parent !== parent");
37-
}
38-
if (parent) {
39-
// Make sure each child is contained within the parent.
40-
if (node.pos < parent.pos) {
41-
throw new Error("node.pos < parent.pos");
42-
}
43-
if (node.end > parent.end) {
44-
throw new Error("node.end > parent.end");
45-
}
46-
}
47-
48-
ts.forEachChild(node, child => {
49-
Test262BaselineRunner.checkInvariants(child, node);
50-
});
51-
52-
// Make sure each of the children is in order.
53-
var currentPos = 0;
54-
ts.forEachChild(node,
55-
child => {
56-
if (child.pos < currentPos) {
57-
throw new Error("child.pos < currentPos");
58-
}
59-
currentPos = child.end;
60-
},
61-
(array: ts.NodeArray<ts.Node>) => {
62-
if (array.pos < node.pos) {
63-
throw new Error("array.pos < node.pos");
64-
}
65-
if (array.end > node.end) {
66-
throw new Error("array.end > node.end");
67-
}
68-
69-
if (array.pos < currentPos) {
70-
throw new Error("array.pos < currentPos");
71-
}
72-
for (var i = 0, n = array.length; i < n; i++) {
73-
if (array[i].pos < currentPos) {
74-
throw new Error("array[i].pos < currentPos");
75-
}
76-
currentPos = array[i].end
77-
}
78-
79-
currentPos = array.end;
80-
});
81-
82-
var childNodesAndArrays: any[] = [];
83-
ts.forEachChild(node, child => { childNodesAndArrays.push(child) }, array => { childNodesAndArrays.push(array) });
84-
85-
for (var childName in node) {
86-
if (childName === "parent" || childName === "nextContainer" || childName === "modifiers" || childName === "externalModuleIndicator") {
87-
continue;
88-
}
89-
var child = (<any>node)[childName];
90-
if (Test262BaselineRunner.isNodeOrArray(child)) {
91-
if (childNodesAndArrays.indexOf(child) < 0) {
92-
throw new Error("Child when forEach'ing over node. " + (<any>ts).SyntaxKind[node.kind] + "-" + childName);
93-
}
94-
}
95-
}
96-
}
97-
}
98-
9924
private static serializeSourceFile(file: ts.SourceFile): string {
10025
function getKindName(k: number): string {
10126
return (<any>ts).SyntaxKind[k]
@@ -264,7 +189,7 @@ class Test262BaselineRunner extends RunnerBase {
264189

265190
it('satisfies invariants', () => {
266191
var sourceFile = testState.checker.getProgram().getSourceFile(Test262BaselineRunner.getTestFilePath(testState.filename));
267-
Test262BaselineRunner.checkInvariants(sourceFile, /*parent:*/ undefined);
192+
Utils.checkInvariants(sourceFile, /*parent:*/ undefined);
268193
});
269194

270195
it('has the expected AST',() => {

‎src/services/services.ts‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1679,7 +1679,7 @@ module ts {
16791679
var scriptSnapshot = this.hostCache.getScriptSnapshot(filename);
16801680

16811681
var start = new Date().getTime();
1682-
sourceFile = createSourceFileFromScriptSnapshot(filename, scriptSnapshot, getDefaultCompilerOptions(), version, /*isOpen*/ true);
1682+
sourceFile = createLanguageServiceSourceFile(filename, scriptSnapshot, getDefaultCompilerOptions(), version, /*isOpen*/ true);
16831683
this.host.log("SyntaxTreeCache.Initialize: createSourceFile: " + (new Date().getTime() - start));
16841684

16851685
var start = new Date().getTime();
@@ -1692,7 +1692,7 @@ module ts {
16921692

16931693
var start = new Date().getTime();
16941694
sourceFile = !editRange
1695-
? createSourceFileFromScriptSnapshot(filename, scriptSnapshot, getDefaultCompilerOptions(), version, /*isOpen*/ true)
1695+
? createLanguageServiceSourceFile(filename, scriptSnapshot, getDefaultCompilerOptions(), version, /*isOpen*/ true)
16961696
: this.currentSourceFile.update(scriptSnapshot, version, /*isOpen*/ true, editRange);
16971697
this.host.log("SyntaxTreeCache.Initialize: updateSourceFile: " + (new Date().getTime() - start));
16981698

@@ -1718,7 +1718,7 @@ module ts {
17181718
}
17191719
}
17201720

1721-
function createSourceFileFromScriptSnapshot(filename: string, scriptSnapshot: IScriptSnapshot, settings: CompilerOptions, version: string, isOpen: boolean) {
1721+
export function createLanguageServiceSourceFile(filename: string, scriptSnapshot: IScriptSnapshot, settings: CompilerOptions, version: string, isOpen: boolean): SourceFile {
17221722
return SourceFileObject.createSourceFileObject(filename, scriptSnapshot, settings.target, version, isOpen);
17231723
}
17241724

@@ -1769,7 +1769,7 @@ module ts {
17691769
var bucket = getBucketForCompilationSettings(compilationSettings, /*createIfMissing*/ true);
17701770
var entry = lookUp(bucket, filename);
17711771
if (!entry) {
1772-
var sourceFile = createSourceFileFromScriptSnapshot(filename, scriptSnapshot, compilationSettings, version, isOpen);
1772+
var sourceFile = createLanguageServiceSourceFile(filename, scriptSnapshot, compilationSettings, version, isOpen);
17731773

17741774
bucket[filename] = entry = {
17751775
sourceFile: sourceFile,

‎tests/cases/unittests/incrementalParser.ts‎

Lines changed: 166 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,166 @@
1+
/// <reference path="..\..\..\src\harness\external\mocha.d.ts" />
2+
/// <reference path="..\..\..\src\compiler\parser.ts" />
3+
4+
module ts {
5+
function withChange(text: IScriptSnapshot, start: number, length: number, newText: string): { text: IScriptSnapshot; textChangeRange: TextChangeRange; } {
6+
var contents = text.getText(0, text.getLength());
7+
var newContents = contents.substr(0, start) + newText + contents.substring(start + length);
8+
9+
return { text: ScriptSnapshot.fromString(newContents), textChangeRange: new TextChangeRange(new TextSpan(start, length), newText.length) }
10+
}
11+
12+
function withInsert(text: IScriptSnapshot, start: number, newText: string): { text: IScriptSnapshot; textChangeRange: TextChangeRange; } {
13+
return withChange(text, start, 0, newText);
14+
}
15+
16+
function withDelete(text: IScriptSnapshot, start: number, length: number): { text: IScriptSnapshot; textChangeRange: TextChangeRange; } {
17+
return withChange(text, start, length, "");
18+
}
19+
20+
// NOTE: 'reusedElements' is the expected count of elements reused from the old tree to the new
21+
// tree. It may change as we tweak the parser. If the count increases then that should always
22+
// be a good thing. If it decreases, that's not great (less reusability), but that may be
23+
// unavoidable. If it does decrease an investigation should be done to make sure that things
24+
// are still ok and we're still appropriately reusing most of the tree.
25+
function compareTrees(oldText: IScriptSnapshot, newText: IScriptSnapshot, textChangeRange: TextChangeRange, expectedReusedElements: number = -1): void {
26+
// Create a tree for the new text, in a non-incremental fashion.
27+
var options: CompilerOptions = {};
28+
options.target = ScriptTarget.ES5;
29+
30+
var newTree = createLanguageServiceSourceFile(/*fileName:*/ "", newText, options, /*version:*/ "0", /*isOpen:*/ true);
31+
Utils.checkInvariants(newTree, /*parent:*/ undefined);
32+
33+
// Create a tree for the new text, in an incremental fashion.
34+
var oldTree = createLanguageServiceSourceFile(/*fileName:*/ "", oldText, options, /*version:*/ "0", /*isOpen:*/ true);
35+
Utils.checkInvariants(oldTree, /*parent:*/ undefined);
36+
37+
var incrementalNewTree = oldTree.update(newText, "1", /*isOpen:*/ true, textChangeRange);
38+
Utils.checkInvariants(incrementalNewTree, /*parent:*/ undefined);
39+
40+
// We should get the same tree when doign a full or incremental parse.
41+
assertStructuralEquals(newTree, incrementalNewTree);
42+
43+
// There should be no reused nodes between two trees that are fully parsed.
44+
Debug.assert(reusedElements(oldTree, newTree) === 0);
45+
46+
if (expectedReusedElements !== -1) {
47+
var actualReusedCount = reusedElements(oldTree, incrementalNewTree);
48+
Debug.assert(actualReusedCount === expectedReusedElements, actualReusedCount + " !== " + expectedReusedElements);
49+
}
50+
}
51+
52+
function assertStructuralEquals(node1: Node, node2: Node) {
53+
if (node1 === node2) {
54+
return;
55+
}
56+
57+
if (!node1 || !node2) {
58+
throw new Error("!node1 || !node2");
59+
}
60+
61+
if (node1.pos !== node2.pos) {
62+
throw new Error("node1.pos !== node2.pos");
63+
}
64+
65+
if (node1.end !== node2.end) {
66+
throw new Error("node1.end !== node2.end");
67+
}
68+
69+
if (node1.kind !== node2.kind) {
70+
throw new Error("node1.kind !== node2.kind");
71+
}
72+
73+
if (node1.flags !== node2.flags) {
74+
throw new Error("node1.flags !== node2.flags");
75+
}
76+
77+
if (node1.parserContextFlags !== node2.parserContextFlags) {
78+
throw new Error("node1.parserContextFlags !== node2.parserContextFlags");
79+
}
80+
81+
forEachChild(node1,
82+
child1 => {
83+
var childName = findChildName(node1, child1);
84+
var child2: Node = (<any>node2)[childName];
85+
86+
assertStructuralEquals(child1, child2);
87+
},
88+
(array1: NodeArray<Node>) => {
89+
var childName = findChildName(node1, array1);
90+
var array2: NodeArray<Node> = (<any>node2)[childName];
91+
92+
assertArrayStructuralEquals(array1, array2);
93+
});
94+
}
95+
96+
function assertArrayStructuralEquals(array1: NodeArray<Node>, array2: NodeArray<Node>) {
97+
if (array1 === array2) {
98+
return;
99+
}
100+
101+
if (!array1 || !array2) {
102+
throw new Error("!array1 || !array2");
103+
}
104+
105+
if (array1.pos !== array2.pos) {
106+
throw new Error("array1.pos !== array2.pos");
107+
}
108+
109+
if (array1.end !== array2.end) {
110+
throw new Error("array1.end !== array2.end");
111+
}
112+
113+
if (array1.length !== array2.length) {
114+
throw new Error("array1.length !== array2.length");
115+
}
116+
117+
for (var i = 0, n = array1.length; i < n; i++) {
118+
assertStructuralEquals(array1[i], array2[i]);
119+
}
120+
}
121+
122+
function findChildName(parent: any, child: any) {
123+
for (var name in parent) {
124+
if (parent.hasOwnProperty(name) && parent[name] === child) {
125+
return name;
126+
}
127+
}
128+
129+
throw new Error("Could not find child in parent");
130+
}
131+
132+
function reusedElements(oldNode: SourceFile, newNode: SourceFile): number {
133+
var allOldElements = collectElements(oldNode);
134+
var allNewElements = collectElements(newNode);
135+
136+
return filter(allOldElements, v => contains(allNewElements, v)).length;
137+
}
138+
139+
function collectElements(node: Node) {
140+
var result: Node[] = [];
141+
visit(node);
142+
return result;
143+
144+
function visit(node: Node) {
145+
result.push(node);
146+
forEachChild(node, visit);
147+
}
148+
}
149+
150+
describe('Incremental',() => {
151+
it('Inserting into method',() => {
152+
var source = "class C {\r\n" +
153+
" public foo1() { }\r\n" +
154+
" public foo2() {\r\n" +
155+
" return 1;\r\n" +
156+
" }\r\n" +
157+
" public foo3() { }\r\n" +
158+
"}";
159+
160+
var oldText = ScriptSnapshot.fromString(source);
161+
var semicolonIndex = source.indexOf(";");
162+
var newTextAndChange = withInsert(oldText, semicolonIndex, " + 1");
163+
compareTrees(oldText, newTextAndChange.text, newTextAndChange.textChangeRange, 0);
164+
});
165+
});
166+
}

0 commit comments

Comments
 (0)