Repository navigation
Expand file tree
/
Copy pathtest_gdpr.py
More file actions
328 lines (272 loc) · 14.4 KB
/
Copy pathtest_gdpr.py
File metadata and controls
328 lines (272 loc) · 14.4 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
"""Tests for the optional runtime GDPR field encryption (ADR-0030).
The cipher used here is a STDLIB-ONLY fake — it deliberately imports no ``cryptography`` library,
proving the core needs no crypto dependency (GR-7). It still honours the full :class:`Cipher`
contract: a random per-call nonce so the same plaintext encrypts differently each time, a
byte-for-byte inverse, and a wrong-key ``decrypt`` that raises. A production cipher (AES-256-GCM via
``cryptography``) is documented in ``babelqueue/gdpr.py``; the contract these tests pin is the same.
"""
from __future__ import annotations
import base64
import hashlib
import json
import os
import unittest
from typing import Any, Dict
from babelqueue import EnvelopeCodec
from babelqueue.exceptions import DecryptError
from babelqueue.gdpr import Cipher, protect, unprotect
from babelqueue.schema import SensitivePath, sensitive_paths
class XorCipher:
"""A reversible, authenticated, STDLIB-ONLY reference :class:`Cipher` for tests.
NOT cryptographically strong — it is a keystream XOR with a keyed digest tag — but it exercises
every property the contract depends on: a random per-call nonce (same plaintext → different
ciphertext), a byte-exact inverse, base64 output that drops into JSON, and a wrong-key/tampered
``decrypt`` that raises (the tag check fails). This is the seam :func:`protect`/:func:`unprotect`
rely on; the production cipher is the caller's AES-GCM.
"""
_NONCE = 16
def __init__(self, key: bytes) -> None:
self._key = key
def _keystream(self, nonce: bytes, length: int) -> bytes:
out = bytearray()
counter = 0
while len(out) < length:
out += hashlib.sha256(self._key + nonce + counter.to_bytes(4, "big")).digest()
counter += 1
return bytes(out[:length])
def _tag(self, nonce: bytes, ciphertext: bytes) -> bytes:
return hashlib.sha256(self._key + b"tag" + nonce + ciphertext).digest()[:16]
def encrypt(self, plaintext: bytes) -> str:
nonce = os.urandom(self._NONCE)
ks = self._keystream(nonce, len(plaintext))
ct = bytes(p ^ k for p, k in zip(plaintext, ks))
sealed = nonce + ct + self._tag(nonce, ct)
return base64.b64encode(sealed).decode("ascii")
def decrypt(self, ciphertext: str) -> bytes:
raw = base64.b64decode(ciphertext)
nonce, body, tag = raw[: self._NONCE], raw[self._NONCE : -16], raw[-16:]
if self._tag(nonce, body) != tag:
raise ValueError("authentication failed (wrong key or tampered ciphertext)")
ks = self._keystream(nonce, len(body))
return bytes(c ^ k for c, k in zip(body, ks))
KEY = b"0123456789abcdef0123456789abcdef"
def _cipher() -> Cipher:
return XorCipher(KEY)
# A schema marking the leaf `email`, the nested `profile.full_name`, the array-item
# `addresses[].line`, and a container `secret_blob` (a whole object marked sensitive).
SCHEMA: Dict[str, Any] = {
"type": "object",
"properties": {
"order_id": {"type": "integer"},
"email": {"type": "string", "x-gdpr-sensitive": "email"},
"profile": {
"type": "object",
"properties": {
"full_name": {"type": "string", "x-gdpr-sensitive": True},
"tier": {"type": "string"},
},
},
"addresses": {
"type": "array",
"items": {
"type": "object",
"properties": {
"line": {"type": "string", "x-gdpr-sensitive": True},
"city": {"type": "string"},
},
},
},
"secret_blob": {"type": "object", "x-gdpr-sensitive": True},
},
}
def _data() -> Dict[str, Any]:
return {
"order_id": 1042,
"email": "alice@example.com",
"profile": {"full_name": "Alice Liddell", "tier": "gold"},
"addresses": [
{"line": "1 Rabbit Hole", "city": "Wonderland"},
{"line": "2 Looking Glass", "city": "Mirror"},
],
"secret_blob": {"pan": "4111111111111111", "exp": 1230},
}
class SensitivePathsTest(unittest.TestCase):
def test_walks_nested_object_array_item_and_container(self) -> None:
paths = sensitive_paths(SCHEMA)
# Sorted by path, with the array-item using the "field[]" segment.
self.assertEqual(
paths,
[
SensitivePath("addresses[].line", ""),
SensitivePath("email", "email"),
SensitivePath("profile.full_name", ""),
SensitivePath("secret_blob", ""),
],
)
def test_root_mark_reports_empty_path(self) -> None:
self.assertEqual(sensitive_paths({"x-gdpr-sensitive": True}), [SensitivePath("", "")])
def test_validation_neutral_shapes_are_ignored(self) -> None:
# false, "", and a number never mark a property (mirrors the Go parser).
schema = {
"type": "object",
"properties": {
"a": {"type": "string", "x-gdpr-sensitive": False},
"b": {"type": "string", "x-gdpr-sensitive": ""},
"c": {"type": "string", "x-gdpr-sensitive": 1},
"d": {"type": "string"},
},
}
self.assertEqual(sensitive_paths(schema), [])
def test_malformed_schema_is_skipped_not_raised(self) -> None:
self.assertEqual(sensitive_paths({"properties": "nope", "items": 5}), [])
class ProtectRoundTripTest(unittest.TestCase):
def test_round_trip_restores_data_byte_for_byte(self) -> None:
original = _data()
data = _data()
protect(data, SCHEMA, _cipher())
# Marked leaves became ciphertext STRINGS; everything else is untouched.
self.assertIsInstance(data["email"], str)
self.assertNotEqual(data["email"], original["email"])
self.assertIsInstance(data["profile"]["full_name"], str)
self.assertIsInstance(data["addresses"][0]["line"], str)
self.assertIsInstance(data["secret_blob"], str) # the whole container is one ciphertext
# Non-sensitive siblings are bit-identical.
self.assertEqual(data["order_id"], 1042)
self.assertEqual(data["profile"]["tier"], "gold")
self.assertEqual(data["addresses"][0]["city"], "Wonderland")
unprotect(data, SCHEMA, _cipher())
self.assertEqual(data, original)
def test_nested_object_only(self) -> None:
schema = {"type": "object", "properties": {"profile": {"type": "object",
"properties": {"full_name": {"type": "string", "x-gdpr-sensitive": True}}}}}
data = {"profile": {"full_name": "Bob", "tier": "silver"}}
protect(data, schema, _cipher())
self.assertNotEqual(data["profile"]["full_name"], "Bob")
self.assertEqual(data["profile"]["tier"], "silver")
unprotect(data, schema, _cipher())
self.assertEqual(data, {"profile": {"full_name": "Bob", "tier": "silver"}})
def test_array_items(self) -> None:
schema = {"type": "object", "properties": {"addresses": {"type": "array", "items":
{"type": "object", "properties": {"line": {"type": "string", "x-gdpr-sensitive": True}}}}}}
data = {"addresses": [{"line": "a", "city": "x"}, {"line": "b", "city": "y"}]}
protect(data, schema, _cipher())
self.assertNotEqual(data["addresses"][0]["line"], "a")
self.assertNotEqual(data["addresses"][1]["line"], "b")
self.assertEqual(data["addresses"][0]["city"], "x")
unprotect(data, schema, _cipher())
self.assertEqual(data, {"addresses": [{"line": "a", "city": "x"}, {"line": "b", "city": "y"}]})
def test_array_leaf_directly_marked(self) -> None:
# A path ending in "[]" marks the array ELEMENTS themselves (e.g. a list of phone strings).
schema = {"type": "object", "properties":
{"phones": {"type": "array", "items": {"type": "string", "x-gdpr-sensitive": True}}}}
self.assertEqual(sensitive_paths(schema), [SensitivePath("phones[]", "")])
data = {"phones": ["+1-555-0100", "+1-555-0199"]}
protect(data, schema, _cipher())
self.assertNotEqual(data["phones"][0], "+1-555-0100")
self.assertNotEqual(data["phones"][1], "+1-555-0199")
unprotect(data, schema, _cipher())
self.assertEqual(data, {"phones": ["+1-555-0100", "+1-555-0199"]})
def test_non_string_values_round_trip_exactly(self) -> None:
# A sensitive leaf need not be a string: number/bool/null/object restore to the same type.
schema = {"type": "object", "properties": {k: {"x-gdpr-sensitive": True}
for k in ("n", "b", "z", "o")}}
data: Dict[str, Any] = {"n": 1230, "b": True, "z": None, "o": {"k": [1, 2, 3]}}
protect(data, schema, _cipher())
for k in ("n", "b", "z", "o"):
self.assertIsInstance(data[k], str)
unprotect(data, schema, _cipher())
self.assertEqual(data, {"n": 1230, "b": True, "z": None, "o": {"k": [1, 2, 3]}})
def test_root_mark_is_skipped_no_addressable_leaf(self) -> None:
# A mark on the root schema has no addressable leaf inside data — a no-op, not an error.
data = {"a": 1}
protect(data, {"x-gdpr-sensitive": True}, _cipher())
self.assertEqual(data, {"a": 1})
class ProtectSkipsTest(unittest.TestCase):
def test_absent_marked_field_is_skipped(self) -> None:
data = {"order_id": 7} # no `email`, `profile`, etc.
protect(data, SCHEMA, _cipher())
self.assertEqual(data, {"order_id": 7})
def test_type_mismatch_is_skipped(self) -> None:
# Schema says addresses is an array of objects, but the message carries a scalar — skip.
data = {"addresses": "not-an-array", "profile": "not-an-object"}
protect(data, SCHEMA, _cipher())
self.assertEqual(data, {"addresses": "not-an-array", "profile": "not-an-object"})
def test_no_marks_or_none_schema_is_noop(self) -> None:
data = _data()
snapshot = json.dumps(data, sort_keys=True)
protect(data, {"type": "object", "properties": {"email": {"type": "string"}}}, _cipher())
protect(data, None, _cipher())
self.assertEqual(json.dumps(data, sort_keys=True), snapshot)
def test_empty_data_is_noop(self) -> None:
data: Dict[str, Any] = {}
protect(data, SCHEMA, _cipher())
self.assertEqual(data, {})
class UnprotectEdgeTest(unittest.TestCase):
def test_non_string_leaf_left_untouched_idempotent(self) -> None:
# Re-running unprotect is a no-op for NON-STRING leaves: once restored to number/bool/null/
# object they are no longer ciphertext candidates, so a second pass leaves them as-is.
schema = {"type": "object", "properties": {k: {"x-gdpr-sensitive": True}
for k in ("n", "b", "z", "o")}}
data: Dict[str, Any] = {"n": 1230, "b": True, "z": None, "o": {"k": 1}}
protect(data, schema, _cipher())
unprotect(data, schema, _cipher()) # first restore
restored = json.dumps(data, sort_keys=True)
unprotect(data, schema, _cipher()) # second pass is a no-op (non-string leaves left alone)
unprotect(data, schema, _cipher()) # and again — still stable
self.assertEqual(json.dumps(data, sort_keys=True), restored)
def test_rerun_on_restored_string_leaf_raises(self) -> None:
# A restored STRING leaf IS a ciphertext candidate, so a naive double-unprotect on a string
# field fails cleanly (it is not the cipher's output) rather than silently corrupting data —
# callers run unprotect exactly once, matching the documented contract.
data = {"email": "alice@example.com"}
protect(data, SCHEMA, _cipher())
unprotect(data, SCHEMA, _cipher())
self.assertEqual(data, {"email": "alice@example.com"})
with self.assertRaises(DecryptError):
unprotect(data, SCHEMA, _cipher())
def test_wrong_key_raises_decrypt_error(self) -> None:
data = _data()
protect(data, SCHEMA, XorCipher(KEY))
with self.assertRaises(DecryptError):
unprotect(data, SCHEMA, XorCipher(b"f" * 32)) # different key → tag check fails
def test_decrypted_non_json_raises_decrypt_error(self) -> None:
# A cipher that opens but yields bytes that are not JSON also fails cleanly as DecryptError —
# the consumer must never feed corrupt-but-decrypted bytes to the handler.
class GarbageCipher:
def encrypt(self, plaintext: bytes) -> str:
return "garbage"
def decrypt(self, ciphertext: str) -> bytes:
return b"\xff\xfe not json"
data = {"email": "ct"}
with self.assertRaises(DecryptError):
unprotect(data, SCHEMA, GarbageCipher())
def test_non_ciphertext_string_raises_decrypt_error(self) -> None:
# A marked leaf that is a plain (never-protected) string is still a string, so unprotect
# tries to open it and fails cleanly rather than silently passing garbage through.
data = {"email": "alice@example.com"}
with self.assertRaises(DecryptError):
unprotect(data, SCHEMA, _cipher())
class FrozenEnvelopeTest(unittest.TestCase):
def test_protected_envelope_still_decodes_and_preserves_frame(self) -> None:
data = _data()
protect(data, SCHEMA, _cipher())
envelope = EnvelopeCodec.make("urn:babel:orders:created", data,
queue="orders", trace_id="trace-xyz")
body = EnvelopeCodec.encode(envelope)
# data stays pure JSON (the ciphertext is just a string) — encode/decode round-trips.
decoded = EnvelopeCodec.decode(body)
self.assertTrue(EnvelopeCodec.accepts(decoded))
self.assertEqual(decoded["meta"]["schema_version"], 1)
self.assertEqual(decoded["trace_id"], "trace-xyz") # GR-4: trace_id untouched
self.assertEqual(decoded["job"], "urn:babel:orders:created")
# The consumer decrypts in place and recovers the original cleartext exactly.
unprotect(decoded["data"], SCHEMA, _cipher())
self.assertEqual(decoded["data"], _data())
def test_ciphertext_is_a_json_string(self) -> None:
# GR-3: a protected value is a JSON string, so an SDK without the key still carries the frame.
data = {"email": "alice@example.com"}
protect(data, SCHEMA, _cipher())
reparsed = json.loads(json.dumps(data))
self.assertIsInstance(reparsed["email"], str)
if __name__ == "__main__":
unittest.main()