Skip to content

Report retry-confirmed failures as regressions, not flakiness #78223

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
TheRealAlexxx wants to merge 1 commit into
base: main
Choose a base branch
from
+28 −0
Open

Report retry-confirmed failures as regressions, not flakiness #78223

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions Tools/Scripts/webkitpy/layout_tests/views/buildbot_results.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,13 @@ def is_expected(result):
add_to_dict_of_lists(passes, 'Expected to fail, but passed', test)
elif enabled_pixel_tests_in_retry and (actual == ['TEXT', 'IMAGE+TEXT'] or actual == ['TEXT', 'MISSING']):
add_to_dict_of_lists(regressions, actual[0], test)
elif results.get('report') == 'FLAKY':
# We group flaky tests by the first actual result we got.
add_to_dict_of_lists(flaky, actual[0], test)
elif results.get('report') == 'REGRESSION':
# A test failing in both the initial and retry runs is a
# regression, even when the result types differ between runs.
add_to_dict_of_lists(regressions, actual[0], test)
elif len(actual) > 1:
# We group flaky tests by the first actual result we got.
add_to_dict_of_lists(flaky, actual[0], test)
Expand Down
21 changes: 21 additions & 0 deletions Tools/Scripts/webkitpy/layout_tests/views/buildbot_results_unittest.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@

from webkitpy.common.host_mock import MockHost
from webkitpy.tool.mocktool import MockOptions
from webkitpy.layout_tests.models import test_expectations
from webkitpy.layout_tests.models import test_run_results
from webkitpy.layout_tests.models import test_run_results_unittest
from webkitpy.layout_tests.views import buildbot_results
Expand Down Expand Up @@ -86,6 +87,26 @@ def test_print_unexpected_results(self):
printer.print_unexpected_results(summary)
self.assertNotEmpty(out)

def test_print_unexpected_results_regression_with_differing_retry_result(self):
port = MockHost().port_factory.get('test', options=MockOptions(pixel_tests=False, world_leaks=False))
printer, out = self.get_printer()

# A test failing in both the initial and retry runs is a regression,
# even when the result types differ between the runs, and must be
# reported as a regression rather than as flakiness.
initial_results = test_run_results_unittest.run_results(port)
initial_results.add(test_run_results_unittest.get_result('passes/text.html', test_expectations.TIMEOUT), False)
retry_results = test_run_results_unittest.run_results(port)
retry_results.add(test_run_results_unittest.get_result('passes/text.html', test_expectations.CRASH), False)
summary = test_run_results.summarize_results(
port, {None: initial_results.expectations}, initial_results, retry_results, enabled_pixel_tests_in_retry=False)
self.assertEqual(summary['num_regressions'], 1)
self.assertEqual(summary['num_flaky'], 0)
printer.print_unexpected_results(summary)
self.assertIn('Regressions: Unexpected', out.getvalue())
self.assertIn('passes/text.html', out.getvalue())
self.assertNotIn('Unexpected flakiness', out.getvalue())

def test_print_unexpected_results_with_options(self):
port = MockHost().port_factory.get('test', options=MockOptions(pixel_tests=False, world_leaks=False))

Expand Down