-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathtest_dataplane_engine.py
More file actions
222 lines (171 loc) · 10.6 KB
/
Copy pathtest_dataplane_engine.py
File metadata and controls
222 lines (171 loc) · 10.6 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
"""The universal ReadPlane engine: the governed read verbs (count/search/get/aggregate) implemented
ONCE over a backend protocol, so every adapter inherits projection, byte-cap-refuse, capability
fallback, and read-side authorization — instead of each hand-rolling reads (and re-introducing floods).
Proven against an in-memory FakeBackend with a tunable capability profile, so the engine's degradation
paths (no server-side filter, no server-side count, no aggregate) are exercised without a live backend.
"""
from __future__ import annotations
import pytest
from datetime import UTC, datetime
from nilscript.dataplane import (
BulkApprovalRequired,
Capabilities,
CapabilityUnsupported,
ExportStore,
FieldSpec,
ReadPlane,
ResultTooLarge,
TargetSchema,
)
NOW = datetime(2026, 6, 26, tzinfo=UTC)
class FakeBackend:
"""In-memory backend with a tunable capability profile. `rows` is the native store; when a
capability is off, the corresponding native method returns None so the engine must degrade."""
def __init__(self, rows: list[dict], schema: TargetSchema) -> None:
self.rows = rows
self._schema = schema
def describe_target(self, target: str) -> TargetSchema | None:
return self._schema if target == self._schema.target else None
def _match(self, row: dict, predicates) -> bool:
for p in predicates:
v = row.get(p.field)
if p.op == "eq" and v != p.value:
return False
if p.op == "ilike" and str(p.value).lower() not in str(v or "").lower():
return False
return True
def fetch(self, target, *, predicates, fields, sort, limit, after_id):
if not self._schema.capabilities.server_filter:
predicates = [] # backend can't filter — engine must do it (and bound the pull)
rows = [r for r in self.rows if self._match(r, predicates)]
rows = [r for r in rows if after_id is None or r["id"] > after_id]
rows.sort(key=lambda r: r["id"])
return rows[:limit]
def count(self, target, *, predicates):
if not self._schema.capabilities.server_filter:
return None # can't count server-side
return sum(1 for r in self.rows if self._match(r, predicates))
def get_one(self, target, record_id, fields):
return next((r for r in self.rows if r["id"] == record_id), None)
def aggregate(self, target, *, predicates, group_by, metrics):
if not self._schema.capabilities.server_aggregate:
return None
groups: dict = {}
for r in self.rows:
if self._match(r, predicates):
key = r.get(group_by)
groups[key] = groups.get(key, 0) + 1
return [{"key": k, "count": v} for k, v in groups.items()]
def _schema(caps: Capabilities = Capabilities()) -> TargetSchema:
return TargetSchema(
target="res.partner",
fields=(
FieldSpec("id", "int", is_key=True),
FieldSpec("name", "str"),
FieldSpec("phone", "str"),
FieldSpec("salary", "float", sensitivity="sensitive"),
),
cardinality="large",
default_projection=("id", "name", "phone"),
capabilities=caps,
)
def _contacts(n: int) -> list[dict]:
return [
{"id": i, "name": f"c{i}", "phone": f"+9745{i:07d}", "salary": 1000 + i, "junk": "z" * 200}
for i in range(n)
]
# ── projection + cap ────────────────────────────────────────────────────────────────────────────
def test_search_applies_the_default_projection_when_none_requested() -> None:
plane = ReadPlane(FakeBackend(_contacts(3), _schema()))
page = plane.search("res.partner", filter=[], fields=None, limit=50)
# default projection is id/name/phone — NOT junk/salary, and never the whole record.
assert page["items"][0] == {"id": 0, "name": "c0", "phone": "+97450000000"}
def test_search_filters_server_side_and_projects() -> None:
plane = ReadPlane(FakeBackend(_contacts(100), _schema()))
page = plane.search(
"res.partner", filter=[{"field": "name", "op": "eq", "value": "c42"}], fields=("name",), limit=50
)
assert [r["id"] for r in page["items"]] == [42]
assert page["items"][0] == {"id": 42, "name": "c42"}
def test_search_refuses_when_a_page_would_exceed_the_cap() -> None:
plane = ReadPlane(FakeBackend(_contacts(20_000), _schema()))
with pytest.raises(ResultTooLarge):
# ask for a huge limit with a wide field set so the page blows the cap → refuse, not truncate
plane.search("res.partner", filter=[], fields=("name", "phone"), limit=20_000)
# ── count ───────────────────────────────────────────────────────────────────────────────────────
def test_count_is_exact_when_the_backend_can_count() -> None:
plane = ReadPlane(FakeBackend(_contacts(41), _schema()))
assert plane.count("res.partner", filter=[]) == {"count": 41}
def test_count_falls_back_to_approximate_when_backend_cannot_count() -> None:
caps = Capabilities(server_filter=False)
plane = ReadPlane(FakeBackend(_contacts(10), _schema(caps)))
out = plane.count("res.partner", filter=[])
assert out["count"] == 10
assert out["approximate"] is True
# ── capability fallback ──────────────────────────────────────────────────────────────────────────
def test_search_filters_in_the_edge_when_backend_cannot_filter() -> None:
caps = Capabilities(server_filter=False)
plane = ReadPlane(FakeBackend(_contacts(50), _schema(caps)))
page = plane.search(
"res.partner", filter=[{"field": "name", "op": "eq", "value": "c7"}], fields=("name",), limit=50
)
assert [r["id"] for r in page["items"]] == [7]
def test_search_refuses_when_unfilterable_set_exceeds_the_bound() -> None:
# backend can't filter and there are more rows than the edge will pull → refuse, never fetch-all.
caps = Capabilities(server_filter=False)
plane = ReadPlane(FakeBackend(_contacts(100_000), _schema(caps)), edge_filter_bound=1000)
with pytest.raises(ResultTooLarge):
plane.search(
"res.partner", filter=[{"field": "name", "op": "eq", "value": "c7"}], fields=("name",), limit=50
)
# ── aggregate ────────────────────────────────────────────────────────────────────────────────────
def test_aggregate_uses_server_side_grouping() -> None:
rows = [{"id": i, "name": f"c{i}", "phone": "", "salary": 0, "country": "QA" if i % 2 else "SA"}
for i in range(10)]
plane = ReadPlane(FakeBackend(rows, _schema()))
out = plane.aggregate("res.partner", filter=[], group_by="country", metrics=("count",))
by = {g["key"]: g["count"] for g in out["groups"]}
assert by == {"SA": 5, "QA": 5}
def test_aggregate_refuses_when_backend_cannot_group() -> None:
caps = Capabilities(server_aggregate=False)
plane = ReadPlane(FakeBackend(_contacts(10), _schema(caps)))
with pytest.raises(CapabilityUnsupported):
plane.aggregate("res.partner", filter=[], group_by="name", metrics=("count",))
# ── get ──────────────────────────────────────────────────────────────────────────────────────────
def test_get_returns_one_lean_record_by_key() -> None:
plane = ReadPlane(FakeBackend(_contacts(5), _schema()))
rec = plane.get("res.partner", record_id=3, fields=("name", "phone"))
assert rec == {"id": 3, "name": "c3", "phone": "+97450000003"}
def test_get_missing_record_returns_none() -> None:
plane = ReadPlane(FakeBackend(_contacts(2), _schema()))
assert plane.get("res.partner", record_id=99, fields=("name",)) is None
# ── read-side authorization ──────────────────────────────────────────────────────────────────────
def test_sensitive_field_is_dropped_without_a_grant_and_redaction_is_noted() -> None:
plane = ReadPlane(FakeBackend(_contacts(1), _schema()))
page = plane.search("res.partner", filter=[], fields=("name", "salary"), limit=50, grant_fields=())
assert "salary" not in page["items"][0] # not leaked
assert "salary" in page.get("redacted", []) # and the omission is declared, not silent
def test_sensitive_field_passes_when_the_grant_allows_it() -> None:
plane = ReadPlane(FakeBackend(_contacts(1), _schema()))
page = plane.search(
"res.partner", filter=[], fields=("name", "salary"), limit=50, grant_fields=("salary",)
)
assert page["items"][0]["salary"] == 1000
# ── export (bulk read → handle, governed) ────────────────────────────────────────────────────────
def test_export_streams_projected_rows_to_a_handle(tmp_path) -> None:
store = ExportStore(root=tmp_path)
plane = ReadPlane(FakeBackend(_contacts(500), _schema()), export_store=store)
handle = plane.export("res.partner", filter=[], fields=("name",), tenant="ws-1", now=NOW)
assert handle.rows == 500
rows = list(store.open(handle.handle, tenant="ws-1", now=NOW))
assert rows[0] == {"id": 0, "name": "c0"} # projected, not the whole record
def test_bulk_export_above_threshold_requires_approval(tmp_path) -> None:
store = ExportStore(root=tmp_path)
plane = ReadPlane(FakeBackend(_contacts(5000), _schema()), export_store=store, bulk_threshold=1000)
with pytest.raises(BulkApprovalRequired):
plane.export("res.partner", filter=[], fields=("name",), tenant="ws-1", now=NOW, approved=False)
def test_bulk_export_proceeds_when_approved(tmp_path) -> None:
store = ExportStore(root=tmp_path)
plane = ReadPlane(FakeBackend(_contacts(5000), _schema()), export_store=store, bulk_threshold=1000)
handle = plane.export("res.partner", filter=[], fields=("name",), tenant="ws-1", now=NOW, approved=True)
assert handle.rows == 5000