Skip to content

Show TestNG before/after group fixtures in reports #1305

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
baev merged 1 commit into from
Jul 7, 2026
Merged

Show TestNG before/after group fixtures in reports #1305

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
97 changes: 97 additions & 0 deletions allure-testng/src/main/java/io/qameta/allure/testng/AllureTestNg.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Deque;
import java.util.HashMap;
import java.util.HashSet;
Expand All @@ -74,6 +75,7 @@
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.function.Consumer;
Expand Down Expand Up @@ -170,6 +172,20 @@ public class AllureTestNg
* Store uuid for data provider scopes.
*/
private final Map<ITestNGMethod, String> dataProviderScopeUuidStorage = new ConcurrentHashMap<>();

/**
* Store uuid for group fixture scopes: one scope per test context and declared group set. Before and after
* group fixtures declaring the same groups share the scope, and the tests belonging to any of those groups
* are its children.
*/
private final Map<GroupsScopeKey, String> groupScopeUuidStorage = new ConcurrentHashMap<>();

/**
* The uuids of the started tests, per test context and group name. An after-groups fixture without a matching
* before-groups fixture creates its scope only after the group tests have already finished and been written,
* so they can no longer be reached through storage and are linked by these recorded uuids instead.
*/
private final Map<GroupKey, List<String>> groupTestUuidStorage = new ConcurrentHashMap<>();
private final ReadWriteLock lock = new ReentrantReadWriteLock();
private final AllureLifecycle lifecycle;
private final AllureTestNgTestFilter testFilter;
Expand Down Expand Up @@ -285,6 +301,15 @@ public void onFinish(final ITestContext context) {
.map(ITestNGMethod::getTestClass)
.distinct()
.forEach(this::onAfterClass);

groupScopeUuidStorage.entrySet().removeIf(entry -> {
if (entry.getKey().context().equals(context)) {
getLifecycle().writeScope(scopeKey(entry.getValue()));
return true;
}
return false;
});
groupTestUuidStorage.keySet().removeIf(key -> key.context().equals(context));
}

public void onBeforeClass(final ITestClass testClass) {
Expand Down Expand Up @@ -328,6 +353,27 @@ public void onTestStart(final ITestResult testResult) {
Optional.of(testResult)
.map(ITestResult::getMethod)
.ifPresent(method -> addTestToDataProviderScope(method, uuid));

addTestToGroupScopes(testResult, uuid);
}

private void addTestToGroupScopes(final ITestResult testResult, final String uuid) {
final String[] groups = testResult.getMethod().getGroups();
if (groups.length == 0) {
return;
}
final ITestContext context = testResult.getTestContext();
Stream.of(groups).forEach(
group -> groupTestUuidStorage
.computeIfAbsent(new GroupKey(context, group), key -> new CopyOnWriteArrayList<>())
.add(uuid)
);
final Set<String> testGroups = Set.copyOf(Arrays.asList(groups));
groupScopeUuidStorage.forEach((key, scopeUuid) -> {
if (key.context().equals(context) && !Collections.disjoint(key.groups(), testGroups)) {
getLifecycle().addTestToScope(scopeKey(scopeUuid), testKey(uuid));
}
});
}

private void linkPendingBeforeMethodScopes(final AllureExternalKey testKey) {
Expand Down Expand Up @@ -508,6 +554,7 @@ public void beforeInvocation(final IInvokedMethod method, final ITestResult test
if (isSupportedConfigurationFixture(testMethod)) {
ifSuiteFixtureStarted(context.getSuite(), testMethod);
ifTestFixtureStarted(context, testMethod);
ifGroupsFixtureStarted(context, testMethod);
ifClassFixtureStarted(testMethod);
ifMethodFixtureStarted(testMethod);
}
Expand Down Expand Up @@ -542,6 +589,36 @@ private void ifTestFixtureStarted(final ITestContext context, final ITestNGMetho
}
}

private void ifGroupsFixtureStarted(final ITestContext context, final ITestNGMethod testMethod) {
if (testMethod.isBeforeGroupsConfiguration()) {
startBefore(getOrCreateGroupsScope(context, testMethod.getBeforeGroups()), testMethod);
}
if (testMethod.isAfterGroupsConfiguration()) {
startAfter(getOrCreateGroupsScope(context, testMethod.getAfterGroups()), testMethod);
}
}

/**
* Returns the uuid of the scope shared by the group fixtures declaring the given groups, registering it on
* first use. A scope first created by an after-groups fixture comes into existence after the tests of its
* groups have already finished and been written, so the tests recorded for those groups are linked
* retroactively by uuid.
*/
private String getOrCreateGroupsScope(final ITestContext context, final String[] groups) {
final GroupsScopeKey key = new GroupsScopeKey(context, Set.copyOf(Arrays.asList(groups)));
return groupScopeUuidStorage.computeIfAbsent(key, k -> {
final String uuid = UUID.randomUUID().toString();
final AllureExternalKey scopeKey = scopeKey(uuid);
getLifecycle().registerScope(scopeKey);
k.groups().stream()
.map(group -> groupTestUuidStorage.getOrDefault(new GroupKey(context, group), List.of()))
.flatMap(List::stream)
.distinct()
.forEach(testUuid -> getLifecycle().addTestToScope(scopeKey, testUuid));
return uuid;
});
}

private void startBefore(final String parentUuid, final ITestNGMethod method) {
final String uuid = currentExecutable.get();
getLifecycle().startBeforeFixture(scopeKey(parentUuid), fixtureKey(uuid), getFixtureResult(method));
Expand Down Expand Up @@ -644,6 +721,12 @@ public void onConfigurationFailure(final ITestResult itr) {
linkTestToScope(getUniqueUuid(itr.getTestContext()), uuid);
linkTestToScope(getUniqueUuid(itr.getTestContext().getSuite()), uuid);
addTestToClassScope(itr.getMethod().getTestClass(), uuid);
if (itr.getMethod().isBeforeGroupsConfiguration()) {
linkTestToScope(getOrCreateGroupsScope(itr.getTestContext(), itr.getMethod().getBeforeGroups()), uuid);
}
if (itr.getMethod().isAfterGroupsConfiguration()) {
linkTestToScope(getOrCreateGroupsScope(itr.getTestContext(), itr.getMethod().getAfterGroups()), uuid);
}
// results created for configuration failure should not be considered as test cases.
getLifecycle().updateTest(
testKey(uuid),
Expand Down Expand Up @@ -744,6 +827,7 @@ private boolean isSupportedConfigurationFixture(final ITestNGMethod testMethod)
return testMethod.isBeforeMethodConfiguration() || testMethod.isAfterMethodConfiguration()
|| testMethod.isBeforeTestConfiguration() || testMethod.isAfterTestConfiguration()
|| testMethod.isBeforeClassConfiguration() || testMethod.isAfterClassConfiguration()
|| testMethod.isBeforeGroupsConfiguration() || testMethod.isAfterGroupsConfiguration()
|| testMethod.isBeforeSuiteConfiguration() || testMethod.isAfterSuiteConfiguration();
}

Expand Down Expand Up @@ -969,4 +1053,17 @@ private static boolean isClassAvailableOnClasspath(final String clazz) {
private record CurrentTest(ITestResult source, String uuid) {
}

/**
* The identity of a group fixture scope: the test context the group configuration methods run in, plus the set
* of group names they declare.
*/
private record GroupsScopeKey(ITestContext context, Set<String> groups) {
}

/**
* A single group within a test context, used to record the tests started for that group.
*/
private record GroupKey(ITestContext context, String group) {
}

}
178 changes: 177 additions & 1 deletion allure-testng/src/test/java/io/qameta/allure/testng/AllureTestNgTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -547,6 +547,181 @@ public void perTestTagFixtures(final XmlSuite.ParallelMode mode, final int threa
assertAfterFixtures(testContainers, after1, after2);
}

@AllureFeatures.Fixtures
@ParameterizedTest
@MethodSource("parallelConfiguration")
@DisplayName("Group fixtures")
public void perGroupFixtures(final XmlSuite.ParallelMode mode, final int threadCount) {
final AllureResults results = runTestNgSuites(
parallel(mode, threadCount),
"suites/per-group-fixtures-combination.xml"
);

assertThat(results.getTestResults())
.extracting(TestResult::getName)
.containsExactlyInAnyOrder("test1", "test2", "testWithoutGroup");

final TestResult test1 = findTestResultByName(results, "test1");
final TestResult test2 = findTestResultByName(results, "test2");

final List<TestResultContainer> containers = results.getTestResultContainers();
assertBeforeFixtures(containers, "beforeGroup");
assertAfterFixtures(containers, "afterGroup");

final List<TestResultContainer> groupScopes = findContainersByFixtureName(containers, "beforeGroup");
assertThat(groupScopes)
.as("Group fixtures declaring the same groups should share a single scope")
.hasSize(1);
final TestResultContainer groupScope = groupScopes.get(0);
assertThat(groupScope.getAfters())
.extracting(FixtureResult::getName)
.containsExactly("afterGroup");
assertThat(groupScope.getChildren())
.as("Group scope should list only the tests belonging to the group")
.containsExactlyInAnyOrder(test1.getUuid(), test2.getUuid());
}

@AllureFeatures.Fixtures
@ParameterizedTest
@MethodSource("parallelConfiguration")
@DisplayName("After groups fixture without matching before groups fixture")
public void afterGroupsFixtureWithoutBeforeGroupsFixture(final XmlSuite.ParallelMode mode,
final int threadCount) {
final AllureResults results = runTestNgSuites(
parallel(mode, threadCount),
"suites/after-group-fixtures.xml"
);

assertThat(results.getTestResults())
.extracting(TestResult::getName)
.containsExactlyInAnyOrder("test1", "test2");

final TestResult test1 = findTestResultByName(results, "test1");
final TestResult test2 = findTestResultByName(results, "test2");

final List<TestResultContainer> containers = results.getTestResultContainers();
assertAfterFixtures(containers, "afterGroup");

final List<TestResultContainer> groupScopes = findContainersByFixtureName(containers, "afterGroup");
assertThat(groupScopes)
.as("After groups fixture should be reported in a single scope")
.hasSize(1);
assertThat(groupScopes.get(0).getChildren())
.as("Tests already finished when the scope appears should be linked to it by uuid")
.containsExactlyInAnyOrder(test1.getUuid(), test2.getUuid());
}

@AllureFeatures.Fixtures
@Test
@DisplayName("After groups fixture declaring multiple groups")
public void afterGroupsFixtureDeclaringMultipleGroups() {
final AllureResults results = runTestNgSuites("suites/after-groups-multiple-groups.xml");

assertThat(results.getTestResults())
.extracting(TestResult::getName)
.containsExactlyInAnyOrder("test1", "test2", "testInBothGroups");

final TestResult test1 = findTestResultByName(results, "test1");
final TestResult test2 = findTestResultByName(results, "test2");
final TestResult testInBothGroups = findTestResultByName(results, "testInBothGroups");

final List<TestResultContainer> containers = results.getTestResultContainers();
assertAfterFixtures(containers, "afterGroups");

final List<TestResultContainer> groupScopes = findContainersByFixtureName(containers, "afterGroups");
assertThat(groupScopes)
.as("Group fixtures declaring the same groups should share a single scope")
.hasSize(1);
assertThat(groupScopes.get(0).getChildren())
.as("Tests from all declared groups should be linked exactly once")
.containsExactlyInAnyOrder(test1.getUuid(), test2.getUuid(), testInBothGroups.getUuid());
}

@AllureFeatures.Fixtures
@Test
@DisplayName("Mixed single-group and multi-group fixtures")
public void mixedGroupFixtures() {
final AllureResults results = runTestNgSuites("suites/mixed-group-fixtures.xml");

assertThat(results.getTestResults())
.extracting(TestResult::getName)
.containsExactlyInAnyOrder("testA", "testB", "testAB", "testBA");

final TestResult testA = findTestResultByName(results, "testA");
final TestResult testB = findTestResultByName(results, "testB");
final TestResult testAB = findTestResultByName(results, "testAB");
final TestResult testBA = findTestResultByName(results, "testBA");

final List<TestResultContainer> containers = results.getTestResultContainers();
assertThat(containers)
.filteredOn(container -> !container.getBefores().isEmpty())
.as("Four group fixtures declaring three distinct group sets should share three scopes")
.hasSize(3);

final List<TestResultContainer> scopeA = findContainersByFixtureName(containers, "beforeGroupA");
assertThat(scopeA).hasSize(1);
assertThat(scopeA.get(0).getBefores())
.extracting(FixtureResult::getName)
.containsExactly("beforeGroupA");
assertThat(scopeA.get(0).getChildren())
.as("Scope of group a should list every test belonging to a")
.containsExactlyInAnyOrder(testA.getUuid(), testAB.getUuid(), testBA.getUuid());

final List<TestResultContainer> scopeB = findContainersByFixtureName(containers, "beforeGroupB");
assertThat(scopeB).hasSize(1);
assertThat(scopeB.get(0).getBefores())
.extracting(FixtureResult::getName)
.containsExactly("beforeGroupB");
assertThat(scopeB.get(0).getChildren())
.as("Scope of group b should list every test belonging to b")
.containsExactlyInAnyOrder(testB.getUuid(), testAB.getUuid(), testBA.getUuid());

final List<TestResultContainer> scopeAB = findContainersByFixtureName(containers, "beforeGroupsAB");
assertThat(scopeAB).hasSize(1);
// TestNG invokes a multi-group before-groups method once per declared group, so the shared
// scope is asserted by fixture names without invocation counts
assertThat(scopeAB.get(0).getBefores())
.as("Fixtures declaring the same groups in different order should share the scope")
.extracting(FixtureResult::getName)
.containsOnly("beforeGroupsAB", "beforeGroupsBA");
assertThat(scopeAB.get(0).getChildren())
.as("Scope of groups a+b should list every test belonging to either group")
.containsExactlyInAnyOrder(
testA.getUuid(), testB.getUuid(), testAB.getUuid(), testBA.getUuid()
);
}

@AllureFeatures.Fixtures
@Issue("953")
@Test
@DisplayName("Suite fixtures are linked directly to tests from all test tags")
public void perSuiteFixturesAcrossTestTags() {
final AllureResults results = runTestNgSuites("suites/suite-fixtures-multiple-tags.xml");

assertThat(results.getTestResults())
.extracting(TestResult::getName)
.containsExactlyInAnyOrder("test", "shouldTest");

final TestResult test = findTestResultByName(results, "test");
final TestResult shouldTest = findTestResultByName(results, "shouldTest");
final List<String> testUuids = asList(test.getUuid(), shouldTest.getUuid());

final List<TestResultContainer> containers = results.getTestResultContainers();

// the report does not resolve nested containers, so every scope must list its tests directly
assertThat(containers)
.allSatisfy(container -> assertThat(container.getChildren()).isSubsetOf(testUuids));

final List<TestResultContainer> suiteScopes = findContainersByFixtureName(containers, "beforeSuite1");
assertThat(suiteScopes).hasSize(1);
assertThat(suiteScopes.get(0).getChildren())
.as("Suite fixtures should apply to the tests of every test tag")
.containsExactlyInAnyOrder(test.getUuid(), shouldTest.getUuid());
assertThat(suiteScopes.get(0).getAfters())
.extracting(FixtureResult::getName)
.containsExactly("afterSuite1", "afterSuite2");
}

@AllureFeatures.SkippedTests
@Test
@DisplayName("Skipped suite")
Expand Down Expand Up @@ -1438,7 +1613,8 @@ private static Stream<Arguments> failedFixtures() {
return Stream.of(
arguments("suites/failed-before-test-fixture.xml", "beforeTest"),
arguments("suites/failed-before-class-fixture.xml", "beforeClass"),
arguments("suites/failed-before-suite-fixture.xml", "beforeSuite")
arguments("suites/failed-before-suite-fixture.xml", "beforeSuite"),
arguments("suites/failed-before-groups-fixture.xml", "beforeGroups")
);
}

Expand Down
Loading
Loading