-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathtest_diff.py
More file actions
154 lines (122 loc) · 5.32 KB
/
Copy pathtest_diff.py
File metadata and controls
154 lines (122 loc) · 5.32 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
"""Unit tests for the diff engine on hand-crafted reports."""
from __future__ import annotations
import json
from profile_diff.diff import (
compare,
render_json,
render_markdown,
render_text,
)
def make_report(label, functions, total, peak):
return {
"schema_version": 1,
"label": label,
"metadata": {"python_version": "3.11.0", "target": "callable x", "top_n": 30},
"total_time_s": total,
"functions": [
{"key": k, "ncalls": 1, "tottime": c, "cumtime": c} for k, c in functions
],
"memory": {"peak_bytes": peak, "top_allocations": []},
}
BASE = make_report(
"base",
[("a.py:1(alpha)", 0.100), ("b.py:2(beta)", 0.050), ("c.py:3(gamma)", 0.010)],
total=0.200,
peak=1_000_000,
)
# alpha regresses (+50%), beta improves (-50%), gamma unchanged, delta is new.
HEAD = make_report(
"head",
[("a.py:1(alpha)", 0.150), ("b.py:2(beta)", 0.025), ("c.py:3(gamma)", 0.010), ("d.py:4(delta)", 0.030)],
total=0.260,
peak=1_500_000,
)
def test_regression_detected():
result = compare(BASE, HEAD, threshold_pct=5.0)
keys = [d.key for d in result.regressions]
assert "a.py:1(alpha)" in keys
reg = next(d for d in result.regressions if d.key == "a.py:1(alpha)")
assert reg.delta > 0
assert abs(reg.delta_pct - 50.0) < 1e-6
assert result.has_regression is True
def test_improvement_detected():
result = compare(BASE, HEAD, threshold_pct=5.0)
keys = [d.key for d in result.improvements]
assert "b.py:2(beta)" in keys
imp = next(d for d in result.improvements if d.key == "b.py:2(beta)")
assert imp.delta < 0
assert abs(imp.delta_pct + 50.0) < 1e-6
def test_new_and_removed_hotspots():
base_only = make_report("base", [("gone.py:1(gone)", 0.02)], 0.02, 10)
head_only = make_report("head", [("fresh.py:1(fresh)", 0.03)], 0.03, 10)
result = compare(base_only, head_only)
assert [d.key for d in result.new_hotspots] == ["fresh.py:1(fresh)"]
assert [d.key for d in result.removed_hotspots] == ["gone.py:1(gone)"]
def test_threshold_pct_filters_small_moves():
# gamma is unchanged (0%), so it must never appear.
result = compare(BASE, HEAD, threshold_pct=5.0)
all_keys = [d.key for d in result.regressions + result.improvements]
assert "c.py:3(gamma)" not in all_keys
# A large threshold suppresses the 50% moves too.
result_high = compare(BASE, HEAD, threshold_pct=75.0)
assert result_high.regressions == []
assert result_high.improvements == []
assert result_high.has_regression is False
def test_min_abs_ms_filters_tiny_absolute_moves():
# alpha moved 50 ms, beta moved 25 ms. min-abs 30 ms keeps only alpha.
result = compare(BASE, HEAD, threshold_pct=5.0, min_abs_ms=30.0)
assert [d.key for d in result.regressions] == ["a.py:1(alpha)"]
assert result.improvements == [] # beta's 25 ms move is filtered out
def test_fail_on_regression_flag_semantics():
# has_regression drives the CLI exit code.
assert compare(BASE, HEAD).has_regression is True
# No regressions when head is strictly faster everywhere.
faster = make_report("head", [("a.py:1(alpha)", 0.05)], 0.05, 500_000)
only_alpha_base = make_report("base", [("a.py:1(alpha)", 0.10)], 0.10, 500_000)
assert compare(only_alpha_base, faster).has_regression is False
def test_stable_ordering_is_deterministic():
r1 = compare(BASE, HEAD, threshold_pct=1.0)
r2 = compare(BASE, HEAD, threshold_pct=1.0)
assert [d.key for d in r1.regressions] == [d.key for d in r2.regressions]
# Regressions sorted by descending delta, ties broken by key.
tied_base = make_report(
"base", [("z.py:1(z)", 0.10), ("a.py:1(a)", 0.10)], 0.20, 1
)
tied_head = make_report(
"head", [("z.py:1(z)", 0.20), ("a.py:1(a)", 0.20)], 0.40, 1
)
result = compare(tied_base, tied_head, threshold_pct=1.0)
# Equal deltas -> alphabetical by key.
assert [d.key for d in result.regressions] == ["a.py:1(a)", "z.py:1(z)"]
def test_top_limits_rows():
funcs_base = [(f"m.py:{i}(f{i})", 0.10) for i in range(20)]
funcs_head = [(f"m.py:{i}(f{i})", 0.30) for i in range(20)]
result = compare(
make_report("base", funcs_base, 2.0, 1),
make_report("head", funcs_head, 6.0, 1),
threshold_pct=1.0,
top=3,
)
assert len(result.regressions) == 3
def test_markdown_contains_marker_and_tables():
md = render_markdown(compare(BASE, HEAD))
assert md.startswith("<!-- profile-diff-action -->")
assert "Top regressions" in md
assert "Top improvements" in md
assert "`a.py:1(alpha)`" in md
# Peak memory summary present with an up-arrow (head uses more).
assert "Peak memory:" in md
assert "▲" in md
def test_text_and_json_renderers():
result = compare(BASE, HEAD)
text = render_text(result)
assert "Performance profile diff" in text
assert "<!-- profile-diff-action -->" not in text # text form has no marker
payload = render_json(result)
assert payload["summary"]["has_regression"] is True
# Round-trips through json.
assert json.loads(json.dumps(payload))["summary"]["total_delta_s"] > 0
def test_empty_diff_message():
same = make_report("x", [("a.py:1(a)", 0.10)], 0.10, 100)
md = render_markdown(compare(same, same))
assert "No function crossed the reporting thresholds" in md