Skip to content

Commit dfd3f61

Browse files
d3r3kkd3r3kk
authored
Update parser for pytest to handle changes in pytest >= 3.7 (while still supporting pytest < 3.7) (microsoft#2465)
1 parent 90d1e4a commit dfd3f61

File tree

6 files changed

+1558
-7
lines changed

6 files changed

+1558
-7
lines changed

‎.vscode/launch.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -129,7 +129,7 @@
129129
"name": "Extension + Debugger",
130130
"configurations": [
131131
"Launch Extension",
132-
"Launch Extension as debugServer"
132+
"Launch Debugger as debugServer"
133133
]
134134
}
135135
]

‎news/2 Fixes/2347.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Fix pytest >= 3.7 test discovery.

‎src/client/unittests/pytest/services/parserService.ts‎

Lines changed: 54 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,9 @@ const DELIMITER = '\'';
1111

1212
@injectable()
1313
export class TestsParser implements ITestsParser {
14+
1415
constructor(@inject(ITestsHelper) private testsHelper: ITestsHelper) { }
16+
1517
public parse(content: string, options: ParserOptions): Tests {
1618
const testFiles = this.getTestFiles(content, options);
1719
return this.testsHelper.flattenTestFiles(testFiles);
@@ -28,13 +30,25 @@ export class TestsParser implements ITestsParser {
2830

2931
let haveErrors = false;
3032

33+
let packagePrefix: string = '';
3134
content.split(/\r?\n/g).forEach((line, index, lines) => {
3235
if (options.token && options.token.isCancellationRequested) {
3336
return;
3437
}
35-
if (line.trim().startsWith('<Module \'') || index === lines.length - 1) {
36-
// process the previous lines
37-
this.parsePyTestModuleCollectionResult(options.cwd, logOutputLines, testFiles, parentNodes);
38+
39+
const trimmedLine: string = line.trim();
40+
41+
if (trimmedLine.startsWith('<Package \'')) {
42+
// Process the previous lines.
43+
this.parsePyTestModuleCollectionResult(options.cwd, logOutputLines, testFiles, parentNodes, packagePrefix);
44+
logOutputLines = [''];
45+
46+
packagePrefix = this.extractPackageName(trimmedLine, options.cwd);
47+
}
48+
49+
if (trimmedLine.startsWith('<Module \'') || index === lines.length - 1) {
50+
// Process the previous lines.
51+
this.parsePyTestModuleCollectionResult(options.cwd, logOutputLines, testFiles, parentNodes, packagePrefix);
3852
logOutputLines = [''];
3953
}
4054
if (errorLine.test(line)) {
@@ -98,15 +112,50 @@ export class TestsParser implements ITestsParser {
98112
return;
99113

100114
}
101-
private parsePyTestModuleCollectionResult(rootDirectory: string, lines: string[], testFiles: TestFile[], parentNodes: { indent: number; item: TestFile | TestSuite }[]) {
115+
116+
/**
117+
* Extract the 'package' name from a given PyTest (>= 3.7) output line.
118+
*
119+
* @param packageLine A single line of output from pytest that starts with `<Package` (may have leading white space).
120+
* @param rootDir Value is pytest's `--rootdir=` parameter.
121+
*/
122+
private extractPackageName(packageLine: string, rootDir: string): string {
123+
const packagePath: string = extractBetweenDelimiters(packageLine, DELIMITER, DELIMITER);
124+
let packageName: string = path.normalize(packagePath);
125+
const tmpRoot: string = path.normalize(rootDir);
126+
127+
if (packageName.indexOf(tmpRoot) === 0) {
128+
packageName = packageName.substring(tmpRoot.length);
129+
if (packageName.startsWith(path.sep)) {
130+
packageName = packageName.substring(1);
131+
}
132+
if (packageName.endsWith(path.sep)) {
133+
packageName = packageName.substring(0, packageName.length - 1);
134+
}
135+
}
136+
packageName = packageName.replace(/\\/g, '/');
137+
return packageName;
138+
}
139+
140+
private parsePyTestModuleCollectionResult(
141+
rootDirectory: string,
142+
lines: string[],
143+
testFiles: TestFile[],
144+
parentNodes: { indent: number; item: TestFile | TestSuite }[],
145+
packagePrefix: string = ''
146+
) {
147+
102148
let currentPackage: string = '';
103149

104150
lines.forEach(line => {
105151
const trimmedLine = line.trim();
106-
const name = extractBetweenDelimiters(trimmedLine, DELIMITER, DELIMITER);
152+
let name: string = extractBetweenDelimiters(trimmedLine, DELIMITER, DELIMITER);
107153
const indent = line.indexOf('<');
108154

109155
if (trimmedLine.startsWith('<Module \'')) {
156+
if (packagePrefix && packagePrefix.length > 0) {
157+
name = packagePrefix.concat('/', name);
158+
}
110159
currentPackage = convertFileToPackage(name);
111160
const fullyQualifiedName = path.isAbsolute(name) ? name : path.resolve(rootDirectory, name);
112161
const testFile = {

‎src/test/unittests/pytest/pytest.discovery.unit.test.ts‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,10 @@ import * as typeMoq from 'typemoq';
1212
import { CancellationToken } from 'vscode';
1313
import { IServiceContainer } from '../../../client/ioc/types';
1414
import { PYTEST_PROVIDER } from '../../../client/unittests/common/constants';
15-
import { ITestDiscoveryService, ITestRunner, ITestsHelper, ITestsParser, Options, TestDiscoveryOptions, Tests } from '../../../client/unittests/common/types';
15+
import {
16+
ITestDiscoveryService, ITestRunner, ITestsHelper,
17+
ITestsParser, Options, TestDiscoveryOptions, Tests
18+
} from '../../../client/unittests/common/types';
1619
import { TestDiscoveryService } from '../../../client/unittests/pytest/services/discoveryService';
1720
import { IArgumentsService, TestFilter } from '../../../client/unittests/types';
1821

@@ -24,6 +27,7 @@ suite('Unit Tests - PyTest - Discovery', () => {
2427
let testParser: typeMoq.IMock<ITestsParser>;
2528
let runner: typeMoq.IMock<ITestRunner>;
2629
let helper: typeMoq.IMock<ITestsHelper>;
30+
2731
setup(() => {
2832
const serviceContainer = typeMoq.Mock.ofType<IServiceContainer>();
2933
argsService = typeMoq.Mock.ofType<IArgumentsService>();

‎src/test/unittests/pytest/pytest.testparser.unit.test.ts‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT License.
3+
4+
'use strict';
5+
6+
import { expect, use } from 'chai';
7+
import * as chaipromise from 'chai-as-promised';
8+
import * as typeMoq from 'typemoq';
9+
import { CancellationToken, OutputChannel, Uri } from 'vscode';
10+
import { getOSType } from '../../..//client/common/platform/osinfo';
11+
import { IApplicationShell, ICommandManager } from '../../../client/common/application/types';
12+
import { OSType } from '../../../client/common/platform/types';
13+
import { IServiceContainer } from '../../../client/ioc/types';
14+
import { TestsHelper } from '../../../client/unittests/common/testUtils';
15+
import { TestFlatteningVisitor } from '../../../client/unittests/common/testVisitors/flatteningVisitor';
16+
import { FlattenedTestFunction, TestDiscoveryOptions, Tests } from '../../../client/unittests/common/types';
17+
import { TestsParser as PyTestsParser } from '../../../client/unittests/pytest/services/parserService';
18+
import { PytestDataPlatformType, pytestScenarioData } from './pytest_unittest_parser_data';
19+
20+
use(chaipromise);
21+
22+
// The PyTest test parsing is done via the stdout result of the
23+
// `pytest --collect-only` command.
24+
//
25+
// There are a few limitations with this approach, the largest issue is mixing
26+
// package and non-package style codebases (stdout does not give subdir
27+
// information of tests in a package when __init__.py is not present).
28+
//
29+
// However, to test all of the various layouts that are available, we have
30+
// created a JSON structure that defines all the tests - see file
31+
// `pytest_unittest_parser_data.ts` in this folder.
32+
suite('Unit Tests - PyTest - Test Parser used in discovery', () => {
33+
34+
// Build tests for the test data that is relevant for this platform.
35+
const testPlatformType: PytestDataPlatformType =
36+
getOSType() === OSType.Windows ?
37+
PytestDataPlatformType.Windows : PytestDataPlatformType.NonWindows;
38+
39+
pytestScenarioData.forEach((testScenario) => {
40+
if (testPlatformType === testScenario.platform) {
41+
42+
const testDescription: string =
43+
`PyTest${testScenario.pytest_version_spec}: ${testScenario.description}`;
44+
45+
test(testDescription, async () => {
46+
// Setup the service container for use by the parser.
47+
const serviceContainer = typeMoq.Mock.ofType<IServiceContainer>();
48+
const appShell = typeMoq.Mock.ofType<IApplicationShell>();
49+
const cmdMgr = typeMoq.Mock.ofType<ICommandManager>();
50+
serviceContainer.setup(s => s.get(typeMoq.It.isValue(IApplicationShell), typeMoq.It.isAny()))
51+
.returns(() => {
52+
return appShell.object;
53+
});
54+
serviceContainer.setup(s => s.get(typeMoq.It.isValue(ICommandManager), typeMoq.It.isAny()))
55+
.returns(() => {
56+
return cmdMgr.object;
57+
});
58+
59+
// Create mocks used in the test discovery setup.
60+
const outChannel = typeMoq.Mock.ofType<OutputChannel>();
61+
const cancelToken = typeMoq.Mock.ofType<CancellationToken>();
62+
cancelToken.setup(c => c.isCancellationRequested).returns(() => false);
63+
const wsFolder = typeMoq.Mock.ofType<Uri>();
64+
65+
// Create the test options for the mocked-up test. All data is either
66+
// mocked or is taken from the JSON test data itself.
67+
const options: TestDiscoveryOptions = {
68+
args: [],
69+
cwd: testScenario.rootdir,
70+
ignoreCache: true,
71+
outChannel: outChannel.object,
72+
token: cancelToken.object,
73+
workspaceFolder: wsFolder.object
74+
};
75+
76+
// Setup the parser.
77+
const testFlattener: TestFlatteningVisitor = new TestFlatteningVisitor();
78+
const testHlp: TestsHelper = new TestsHelper(testFlattener, serviceContainer.object);
79+
const parser = new PyTestsParser(testHlp);
80+
81+
// Each test scenario has a 'stdout' member that is an array of
82+
// stdout lines. Join them here such that the parser can operate
83+
// on stdout-like data.
84+
const stdout: string = testScenario.stdout.join('\n');
85+
86+
const parsedTests: Tests = parser.parse(stdout, options);
87+
88+
// Now we can actually perform tests.
89+
expect(parsedTests).is.not.equal(
90+
undefined,
91+
'Should have gotten tests extracted from the parsed pytest result content.');
92+
93+
expect(parsedTests.testFunctions.length).equals(
94+
testScenario.functionCount,
95+
`Parsed pytest summary contained ${testScenario.functionCount} test functions.`);
96+
97+
testScenario.test_functions.forEach((funcName: string) => {
98+
const findAllTests: FlattenedTestFunction[] | undefined = parsedTests.testFunctions.filter(
99+
(tstFunc: FlattenedTestFunction) => {
100+
return tstFunc.testFunction.nameToRun === funcName;
101+
});
102+
// Each test identified in the testScenario should exist once and only once.
103+
expect(findAllTests).is.not.equal(undefined, `Could not find "${funcName}" in tests.`);
104+
expect(findAllTests.length).is.equal(1, 'There should be exactly one instance of each test.');
105+
});
106+
107+
});
108+
}
109+
});
110+
});

0 commit comments

Comments
 (0)