-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathtest_analysis.py
More file actions
217 lines (172 loc) · 6.73 KB
/
Copy pathtest_analysis.py
File metadata and controls
217 lines (172 loc) · 6.73 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
"""Unit tests for the pure scoring / aggregation / classification logic.
None of these run a real pytest process; they feed synthetic per-run data
straight into the analysis functions.
"""
from __future__ import annotations
from flaky_hunter.analysis import (
NONDETERMINISTIC,
ORDER_DEPENDENT,
STABLE,
TIMING,
RunRecord,
TestObservation,
aggregate,
classify_cause,
)
def obs(outcome, duration=0.0, predecessors=()):
return TestObservation(outcome, duration, frozenset(predecessors))
# --- flakiness scoring & aggregation ---------------------------------------
def test_perfectly_stable_test_scores_zero():
records = [RunRecord({"t::a": "PASSED"}) for _ in range(5)]
report = aggregate(records)
(a,) = report.tests
assert a.flakiness_score == 0.0
assert a.stable is True
assert report.stability_score == 1.0
assert report.flaky_count == 0
def test_flakiness_score_is_fraction_disagreeing_with_mode():
# 7 PASS, 3 FAIL -> modal PASS, 3/10 disagree.
outcomes = ["PASSED"] * 7 + ["FAILED"] * 3
records = [RunRecord({"t::a": o}) for o in outcomes]
(a,) = aggregate(records).tests
assert a.modal_outcome == "PASSED"
assert abs(a.flakiness_score - 0.3) < 1e-9
assert a.breakdown() == "7xPASS 3xFAIL"
def test_suite_stability_is_fraction_of_stable_tests():
records = []
for i in range(4):
records.append(
RunRecord(
{
"t::stable": "PASSED",
"t::flaky": "PASSED" if i % 2 else "FAILED",
}
)
)
report = aggregate(records)
assert report.total_tests == 2
assert report.flaky_count == 1
assert report.stability_score == 0.5
# flaky test ranks first
assert report.tests[0].nodeid == "t::flaky"
def test_modal_outcome_tie_break_is_deterministic():
records = [RunRecord({"t::a": "FAILED"}), RunRecord({"t::a": "PASSED"})]
# 1 each -> alphabetical tie-break picks "FAILED"
(a,) = aggregate(records).tests
assert a.modal_outcome == "FAILED"
def test_aggregate_handles_test_missing_from_some_runs():
records = [
RunRecord({"t::a": "PASSED", "t::b": "PASSED"}),
RunRecord({"t::a": "FAILED"}), # b absent this run
]
report = aggregate(records)
by_id = {t.nodeid: t for t in report.tests}
assert by_id["t::b"].runs == 1
assert by_id["t::b"].stable is True
assert by_id["t::a"].runs == 2
# --- cause: stable ----------------------------------------------------------
def test_classify_stable():
result = classify_cause([obs("PASSED"), obs("PASSED")])
assert result.cause == STABLE
assert result.confidence == 0.0
# --- cause: nondeterministic ------------------------------------------------
def test_classify_nondeterministic_under_fixed_order():
# Same predecessors every run, outcome flips -> not order, not timing.
preds = ("t::x", "t::y")
observations = [
obs("PASSED", 0.01, preds),
obs("FAILED", 0.01, preds),
obs("PASSED", 0.01, preds),
obs("FAILED", 0.01, preds),
]
result = classify_cause(observations)
assert result.cause == NONDETERMINISTIC
assert result.signals["order"] == 0.0
assert result.signals["timing"] == 0.0
# --- cause: order-dependent -------------------------------------------------
def test_classify_order_dependent_names_polluter():
# target fails exactly when "t::polluter" precedes it; order varies.
observations = [
obs("FAILED", 0.01, ("t::polluter", "t::other")),
obs("PASSED", 0.01, ("t::other",)),
obs("FAILED", 0.01, ("t::other", "t::polluter")),
obs("PASSED", 0.01, ()),
obs("FAILED", 0.01, ("t::polluter",)),
obs("PASSED", 0.01, ("t::other",)),
]
result = classify_cause(observations)
assert result.cause == ORDER_DEPENDENT
assert result.candidates == ["t::polluter"]
assert result.confidence >= 0.6
def test_order_signal_ignored_when_order_never_varies():
# Predecessors identical every run -> cannot be order, so nondeterministic.
preds = ("t::a",)
observations = [obs("FAILED", 0.0, preds), obs("PASSED", 0.0, preds)]
result = classify_cause(observations)
assert result.cause == NONDETERMINISTIC
# --- cause: timing ----------------------------------------------------------
def test_classify_timing_when_failures_are_slow():
# Fixed order (no order signal); failures are ~50x slower than passes.
preds = ("t::x",)
observations = [
obs("PASSED", 0.01, preds),
obs("PASSED", 0.01, preds),
obs("FAILED", 0.60, preds),
obs("PASSED", 0.01, preds),
obs("FAILED", 0.55, preds),
]
result = classify_cause(observations)
assert result.cause == TIMING
assert result.signals["timing"] > 0.2
def test_timing_not_triggered_when_durations_stable():
preds = ("t::x",)
observations = [
obs("PASSED", 0.10, preds),
obs("FAILED", 0.10, preds),
obs("PASSED", 0.10, preds),
]
result = classify_cause(observations)
assert result.cause == NONDETERMINISTIC
assert result.signals["timing"] == 0.0
# --- precedence: order beats timing ----------------------------------------
def test_order_takes_precedence_over_timing():
# Both a perfect polluter correlation AND slow-on-fail; order should win
# because it is the more actionable diagnosis.
observations = [
obs("FAILED", 0.9, ("t::polluter",)),
obs("PASSED", 0.01, ()),
obs("FAILED", 0.8, ("t::polluter", "t::z")),
obs("PASSED", 0.01, ("t::z",)),
]
result = classify_cause(observations)
assert result.cause == ORDER_DEPENDENT
# --- end-to-end aggregation with order info --------------------------------
def test_aggregate_classifies_from_run_records():
# Build records where t::target fails iff t::pol ran before it.
records = [
RunRecord(
outcomes={"t::pol": "PASSED", "t::target": "FAILED"},
durations={},
order=["t::pol", "t::target"],
),
RunRecord(
outcomes={"t::pol": "PASSED", "t::target": "PASSED"},
durations={},
order=["t::target", "t::pol"],
),
RunRecord(
outcomes={"t::pol": "PASSED", "t::target": "FAILED"},
durations={},
order=["t::pol", "t::target"],
),
RunRecord(
outcomes={"t::pol": "PASSED", "t::target": "PASSED"},
durations={},
order=["t::target", "t::pol"],
),
]
report = aggregate(records)
target = next(t for t in report.tests if t.nodeid == "t::target")
assert target.cause == ORDER_DEPENDENT
assert "t::pol" in target.candidates
assert report.tests[0].nodeid == "t::target" # flakiest first