Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions build/test-requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -38,3 +38,6 @@ pytest-describe

# for pytest-ruff related tests
pytest-ruff

# for pytest-xdist related tests
pytest-xdist
28 changes: 28 additions & 0 deletions python_files/tests/pytestadapter/.data/xdist_group_tests.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License.

import pytest

# With --dist=loadgroup, pytest-xdist adds "@group_a" to the ids of the grouped tests.


@pytest.mark.xdist_group("group_a")
def test_grouped(): # test_marker--test_grouped
assert True


# The first parameter id contains "@" to check that only the group suffix is removed.
@pytest.mark.xdist_group("group_a")
@pytest.mark.parametrize("email", ["user@example.com", "no-at-sign"])
def test_grouped_param(email): # test_marker--test_grouped_param
assert "@" in email


@pytest.mark.xdist_group("group_a")
@pytest.mark.skip(reason="Skipped to check that skipped results use the original test id")
def test_grouped_skipped(): # test_marker--test_grouped_skipped
assert True


def test_ungrouped(): # test_marker--test_ungrouped
assert True
58 changes: 58 additions & 0 deletions python_files/tests/pytestadapter/expected_execution_test_output.py
Original file line number Diff line number Diff line change
Expand Up @@ -747,3 +747,61 @@
"subtest": None,
}
}

# This is the expected output for the xdist_group_tests.py file when run with --dist=loadgroup.
# └── xdist_group_tests.py
# └── test_grouped: success
# └── test_grouped_param[user@example.com]: success
# └── test_grouped_param[no-at-sign]: failure
# └── test_grouped_skipped: skipped
# └── test_ungrouped: success
xdist_group_tests_path = TEST_DATA_PATH / "xdist_group_tests.py"
xdist_group_expected_execution_output = {
get_absolute_test_id("xdist_group_tests.py::test_grouped", xdist_group_tests_path): {
"test": get_absolute_test_id("xdist_group_tests.py::test_grouped", xdist_group_tests_path),
"outcome": "success",
"message": None,
"traceback": None,
"subtest": None,
},
get_absolute_test_id(
"xdist_group_tests.py::test_grouped_param[user@example.com]", xdist_group_tests_path
): {
"test": get_absolute_test_id(
"xdist_group_tests.py::test_grouped_param[user@example.com]", xdist_group_tests_path
),
"outcome": "success",
"message": None,
"traceback": None,
"subtest": None,
},
get_absolute_test_id(
"xdist_group_tests.py::test_grouped_param[no-at-sign]", xdist_group_tests_path
): {
"test": get_absolute_test_id(
"xdist_group_tests.py::test_grouped_param[no-at-sign]", xdist_group_tests_path
),
"outcome": "failure",
"message": "ERROR MESSAGE",
"traceback": None,
"subtest": None,
},
get_absolute_test_id("xdist_group_tests.py::test_grouped_skipped", xdist_group_tests_path): {
"test": get_absolute_test_id(
"xdist_group_tests.py::test_grouped_skipped", xdist_group_tests_path
),
"outcome": "skipped",
"message": None,
"traceback": None,
"subtest": None,
},
get_absolute_test_id("xdist_group_tests.py::test_ungrouped", xdist_group_tests_path): {
"test": get_absolute_test_id(
"xdist_group_tests.py::test_ungrouped", xdist_group_tests_path
),
"outcome": "success",
"message": None,
"traceback": None,
"subtest": None,
},
}
33 changes: 33 additions & 0 deletions python_files/tests/pytestadapter/test_execution.py
Original file line number Diff line number Diff line change
Expand Up @@ -272,3 +272,36 @@ def test_symlink_run():
# Print the actual_item in JSON format if an assertion fails
print(json.dumps(actual_item, indent=4))
pytest.fail(str(e))


@pytest.mark.skipif(
sys.platform == "win32",
reason="The test pipe on Windows accepts one connection, but pytest-xdist workers connect too",
)
def test_xdist_group_execution():
"""Test pytest execution of tests marked with xdist_group when run with --dist=loadgroup.

pytest-xdist adds "@<group>" to the ids of these tests on its workers. The results must be sent
with the ids found during discovery so that the test explorer can match them.
"""
pytest.importorskip("xdist")
file_path = TEST_DATA_PATH / "xdist_group_tests.py"
actual = runner(["-n", "1", "--dist=loadgroup", os.fspath(file_path)])
assert actual
actual_result_dict = {}
for actual_item in actual:
assert all(item in actual_item for item in ("status", "cwd", "result"))
assert actual_item.get("status") == "success"
actual_result_dict.update(actual_item["result"])
assert not [test_id for test_id in actual_result_dict if test_id.endswith("@group_a")]
# The xdist controller sends each result again under an id without the file path, which
# doesn't match any test. Only compare the results sent by the worker.
file_results = {
test_id: result
for test_id, result in actual_result_dict.items()
if test_id.startswith(f"{os.fspath(file_path)}::")
}
for result in file_results.values():
if result["outcome"] == "failure":
result["message"] = "ERROR MESSAGE"
assert file_results == expected_execution_test_output.xdist_group_expected_execution_output
29 changes: 28 additions & 1 deletion python_files/tests/pytestadapter/test_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,19 @@
import sys
import tempfile

import pytest

from .helpers import (
TEST_DATA_PATH,
)

script_dir = pathlib.Path(__file__).parent.parent.parent
sys.path.append(os.fspath(script_dir))
from vscode_pytest import cached_fsdecode, has_symlink_parent # noqa: E402
from vscode_pytest import ( # noqa: E402
cached_fsdecode,
has_symlink_parent,
remove_xdist_group_suffix,
)


def test_has_symlink_parent_with_symlink():
Expand Down Expand Up @@ -55,3 +61,24 @@ def test_cached_fsdecode():
result3 = cached_fsdecode(test_path2)
assert result3 == os.fspath(test_path2)
assert result3 != result1


@pytest.mark.parametrize(
("test_id", "keywords", "expected"),
[
("test_file.py::test_a@group_a", {"xdist_group": 1}, "test_file.py::test_a"),
(
"test_file.py::TestClass::test_a@group_a",
{"xdist_group": 1},
"test_file.py::TestClass::test_a",
),
("test_file.py::test_a[x@y]@group_a", {"xdist_group": 1}, "test_file.py::test_a[x@y]"),
# Without --dist=loadgroup there is no suffix, but parameter ids can still contain "@".
("test_file.py::test_a[x@y]", {"xdist_group": 1}, "test_file.py::test_a[x@y]"),
("test_file.py::test_a", {"xdist_group": 1}, "test_file.py::test_a"),
# Tests without the xdist_group marker are left untouched.
("test_file.py::test_a@group_a", {}, "test_file.py::test_a@group_a"),
],
)
def test_remove_xdist_group_suffix(test_id, keywords, expected):
assert remove_xdist_group_suffix(test_id, keywords) == expected
33 changes: 30 additions & 3 deletions python_files/vscode_pytest/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
Dict,
Generator,
Literal,
Mapping,
Protocol,
TypedDict,
cast,
Expand Down Expand Up @@ -206,7 +207,9 @@ def pytest_exception_interact(node, call, report):
report_value = "error"
if call.excinfo.typename == "AssertionError":
report_value = "failure"
node_id = get_absolute_test_id(node.nodeid, get_node_path(node))
node_id = get_absolute_test_id(
remove_xdist_group_suffix(node.nodeid, node.keywords), get_node_path(node)
)
if node_id not in collected_tests_so_far:
collected_tests_so_far.add(node_id)
item_result = create_test_outcome(
Expand Down Expand Up @@ -256,6 +259,26 @@ def get_absolute_test_id(test_id: str, test_path: pathlib.Path) -> str:
return "::".join([str(test_path), *split_id])


def remove_xdist_group_suffix(test_id: str, keywords: Mapping[str, Any]) -> str:
"""A function that returns the test id without the group suffix added by pytest-xdist.

With --dist=loadgroup, pytest-xdist renames tests marked with xdist_group to "<test id>@<group>"
on its workers so that each group runs on a single worker. Discovery runs without xdist, so
results must be sent with the original test id for the test explorer to find the test.

Keyword arguments:
test_id -- the pytest id of the test, which may end with the group suffix.
keywords -- the keywords of the test, used to check that it has the xdist_group marker.
"""
if "xdist_group" in keywords:
suffix_start = test_id.rfind("@")
# Parametrize ids can contain "@" too, so only strip an "@" that comes after them,
# the same way pytest-xdist reads the group name back from the test id.
if suffix_start > test_id.rfind("]"):
return test_id[:suffix_start]
return test_id


def pytest_keyboard_interrupt(excinfo):
"""A pytest hook that is called when a keyboard interrupt is raised.

Expand Down Expand Up @@ -331,7 +354,9 @@ def pytest_report_teststatus(report, config): # noqa: ARG001
except KeyError:
node_path = cwd
# Calculate the absolute test id and use this as the ID moving forward.
absolute_node_id = get_absolute_test_id(report.nodeid, node_path)
absolute_node_id = get_absolute_test_id(
remove_xdist_group_suffix(report.nodeid, report.keywords), node_path
)
if absolute_node_id not in collected_tests_so_far:
collected_tests_so_far.add(absolute_node_id)
item_result = create_test_outcome(
Expand Down Expand Up @@ -363,7 +388,9 @@ def pytest_runtest_protocol(item, nextitem): # noqa: ARG001
map_id_to_path[item.nodeid] = get_node_path(item)
skipped = check_skipped_wrapper(item)
if skipped:
absolute_node_id = get_absolute_test_id(item.nodeid, get_node_path(item))
absolute_node_id = get_absolute_test_id(
remove_xdist_group_suffix(item.nodeid, item.keywords), get_node_path(item)
)
report_value = "skipped"
cwd = pathlib.Path.cwd()
if absolute_node_id not in collected_tests_so_far:
Expand Down