-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathexport.py
More file actions
100 lines (82 loc) · 3.8 KB
/
Copy pathexport.py
File metadata and controls
100 lines (82 loc) · 3.8 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
"""export → data-handle subsystem: stream a bulk read to a tenant-scoped artifact on disk and return a
small HANDLE. The rows never enter the agent's context — they are reached only through code in the
sandbox (pandas/DuckDB/sqlite). Handles are tenant-scoped (no cross-tenant read), TTL-expiring, and
PII-at-rest (sandbox-local file, never logged).
This is what makes "give me ALL the data / analyse all 1M rows" possible without a flood: the only
thing that crosses into context is `{handle, format, rows, bytes, schema, expires_at}`.
"""
from __future__ import annotations
import json
import uuid
from collections.abc import Iterable, Iterator
from dataclasses import dataclass
from datetime import datetime, timedelta
from pathlib import Path
from typing import Any
class HandleExpired(Exception):
"""The export artifact's TTL has passed. Surfaced as `HANDLE_EXPIRED` — re-export to get fresh."""
code = "HANDLE_EXPIRED"
class NotAuthorizedHandle(Exception):
"""A tenant tried to open a handle it does not own. Surfaced as `NOT_AUTHORIZED` — handles never
cross tenant boundaries (the bulk-export artifact is PII at rest)."""
code = "NOT_AUTHORIZED"
@dataclass(frozen=True)
class ExportHandle:
"""The small pointer that crosses into context in place of the rows."""
handle: str
format: str
rows: int
bytes: int
schema: dict[str, Any]
expires_at: datetime
class ExportStore:
"""Materialises bulk reads to per-tenant artifacts and serves them back, access-controlled."""
def __init__(self, root: Path) -> None:
self._root = Path(root)
self._root.mkdir(parents=True, exist_ok=True)
self._meta: dict[str, dict[str, Any]] = {}
def _path(self, tenant: str, handle: str) -> Path:
tdir = self._root / tenant
tdir.mkdir(parents=True, exist_ok=True)
return tdir / f"{handle}.{self._meta[handle]['format']}"
def write(
self,
rows: Iterable[dict[str, Any]],
*,
fmt: str,
schema: dict[str, Any],
tenant: str,
now: datetime,
ttl_seconds: int,
) -> ExportHandle:
"""Stream `rows` to a tenant-scoped JSONL artifact; return a small handle (never the rows)."""
handle = uuid.uuid4().hex
self._meta[handle] = {"format": fmt, "tenant": tenant}
path = self._path(tenant, handle)
count = 0
size = 0
with path.open("w", encoding="utf-8") as fh:
for row in rows: # streamed: one row in memory at a time, never the whole set
line = json.dumps(row, ensure_ascii=False, separators=(",", ":"))
fh.write(line + "\n")
count += 1
size += len(line.encode("utf-8")) + 1
expires_at = now + timedelta(seconds=ttl_seconds)
self._meta[handle].update({"rows": count, "bytes": size, "expires_at": expires_at})
return ExportHandle(
handle=handle, format=fmt, rows=count, bytes=size, schema=schema, expires_at=expires_at
)
def open(self, handle: str, *, tenant: str, now: datetime) -> Iterator[dict[str, Any]]:
"""Stream the artifact's rows back to the OWNING tenant, refusing a foreign tenant or an
expired handle. Used by the sandbox bridge to land the file for code execution."""
meta = self._meta.get(handle)
if meta is None or meta["tenant"] != tenant:
raise NotAuthorizedHandle(f"handle {handle} is not readable by tenant {tenant}")
if now >= meta["expires_at"]:
raise HandleExpired(f"handle {handle} expired at {meta['expires_at'].isoformat()}")
path = self._path(tenant, handle)
with path.open("r", encoding="utf-8") as fh:
for line in fh:
line = line.strip()
if line:
yield json.loads(line)