-
Notifications
You must be signed in to change notification settings - Fork 6
Expand file tree
/
Copy pathbinary-detection.ts
More file actions
391 lines (342 loc) · 12.7 KB
/
Copy pathbinary-detection.ts
File metadata and controls
391 lines (342 loc) · 12.7 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
/**
* Binary Detection & File Saving Utilities
*
* Shared binary content detection used by guardLargeResult() to handle
* binary data across all tool result paths (API tools, MCP tools, backend adapters).
*
* Extracted from api-tools.ts for centralized use.
*/
import { mkdirSync, writeFileSync } from 'fs';
import { join } from 'path';
// ============================================================
// Constants
// ============================================================
/** Maximum file size for binary downloads (500MB) — prevents OOM */
export const MAX_DOWNLOAD_SIZE = 500 * 1024 * 1024;
/**
* Magic bytes (file signatures) for common binary formats.
* Used to detect file type when MIME type is unknown or generic.
*/
const MAGIC_SIGNATURES: Array<{ bytes: number[]; ext: string }> = [
{ bytes: [0x25, 0x50, 0x44, 0x46], ext: '.pdf' }, // %PDF
{ bytes: [0x89, 0x50, 0x4E, 0x47], ext: '.png' }, // .PNG
{ bytes: [0xFF, 0xD8, 0xFF], ext: '.jpg' }, // JPEG
{ bytes: [0x47, 0x49, 0x46, 0x38], ext: '.gif' }, // GIF8
{ bytes: [0x50, 0x4B, 0x03, 0x04], ext: '.zip' }, // PK.. (also docx, xlsx, pptx)
{ bytes: [0x52, 0x61, 0x72, 0x21], ext: '.rar' }, // Rar!
{ bytes: [0x1F, 0x8B], ext: '.gz' }, // gzip
{ bytes: [0x42, 0x4D], ext: '.bmp' }, // BM
{ bytes: [0x49, 0x44, 0x33], ext: '.mp3' }, // ID3 (MP3 with ID3 tag)
{ bytes: [0xFF, 0xFB], ext: '.mp3' }, // MP3 frame sync
{ bytes: [0x4F, 0x67, 0x67, 0x53], ext: '.ogg' }, // OggS
{ bytes: [0x66, 0x4C, 0x61, 0x43], ext: '.flac' }, // fLaC
];
/**
* MIME type to file extension mapping for binary downloads.
*/
export const MIME_TO_EXT: Record<string, string> = {
'application/pdf': '.pdf',
'application/zip': '.zip',
'application/gzip': '.gz',
'application/x-gzip': '.gz',
'application/x-tar': '.tar',
'application/x-rar-compressed': '.rar',
'application/x-7z-compressed': '.7z',
'image/png': '.png',
'image/jpeg': '.jpg',
'image/gif': '.gif',
'image/webp': '.webp',
'image/svg+xml': '.svg',
'image/x-icon': '.ico',
'image/bmp': '.bmp',
'image/tiff': '.tiff',
'audio/mpeg': '.mp3',
'audio/wav': '.wav',
'audio/ogg': '.ogg',
'audio/flac': '.flac',
'video/mp4': '.mp4',
'video/webm': '.webm',
'video/quicktime': '.mov',
'video/x-msvideo': '.avi',
'application/msword': '.doc',
'application/vnd.openxmlformats-officedocument.wordprocessingml.document': '.docx',
'application/vnd.ms-excel': '.xls',
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': '.xlsx',
'application/vnd.ms-powerpoint': '.ppt',
'application/vnd.openxmlformats-officedocument.presentationml.presentation': '.pptx',
'application/octet-stream': '.bin',
};
// ============================================================
// Detection Functions
// ============================================================
/**
* Inspect buffer contents to detect binary data.
* Checks for null bytes and high ratio of non-printable characters.
*
* UTF-8 handling: We skip ALL bytes >= 0x80 (multibyte sequences) to avoid
* misclassifying international text (accented chars, emojis, CJK) as binary.
* Only ASCII bytes (0x00-0x7F) are analyzed for printability.
*/
export function looksLikeBinary(buffer: Buffer): boolean {
// Check first 8KB for binary indicators
const sample = buffer.slice(0, 8192);
// Null bytes are a dead giveaway for binary
if (sample.includes(0x00)) return true;
// Count non-printable ASCII characters (skip UTF-8 multibyte entirely)
let nonPrintable = 0;
let asciiCount = 0;
for (const byte of sample) {
// Skip UTF-8 multibyte sequences (both leading and continuation bytes)
if (byte >= 0x80) continue;
asciiCount++;
// Check if ASCII byte is non-printable (excluding common whitespace)
if (byte < 0x09 || (byte > 0x0D && byte < 0x20)) {
nonPrintable++;
}
}
// If >10% of ASCII bytes are non-printable, likely binary
return asciiCount > 0 && (nonPrintable / asciiCount) > 0.10;
}
/**
* Detect file extension from magic bytes (file signature).
* Inspects first bytes of buffer to identify common file formats.
* Returns extension with dot (e.g., '.pdf') or empty string if unknown.
*/
export function detectExtensionFromMagic(buffer: Buffer): string {
if (buffer.length < 8) return '';
// RIFF is a shared container (WAV, WebP, AVI); the four-character form tag at
// bytes 8-11 is what actually distinguishes them, so the "RIFF" prefix alone
// is ambiguous and must not be assumed to be WAV.
if (
buffer.length >= 12 &&
buffer[0] === 0x52 &&
buffer[1] === 0x49 &&
buffer[2] === 0x46 &&
buffer[3] === 0x46
) {
const form = buffer.toString('ascii', 8, 12);
if (form === 'WEBP') return '.webp';
if (form === 'AVI ') return '.avi';
if (form === 'WAVE') return '.wav';
return '';
}
for (const sig of MAGIC_SIGNATURES) {
if (sig.bytes.every((byte, i) => buffer[i] === byte)) {
return sig.ext;
}
}
return '';
}
/**
* Get file extension from MIME type, with optional magic byte fallback.
*/
export function getMimeExtension(mimeType: string | null, buffer?: Buffer): string {
if (mimeType) {
const normalized = (mimeType.toLowerCase().split(';')[0] ?? '').trim();
const ext = MIME_TO_EXT[normalized];
if (ext && ext !== '.bin') return ext;
}
if (buffer) {
return detectExtensionFromMagic(buffer);
}
return '';
}
// ============================================================
// Inline Base64 Detection
// ============================================================
/** Minimum base64 payload length to consider (avoids short tokens, API keys, JWTs) */
const MIN_BASE64_LENGTH = 256;
/** Minimum decoded size in bytes to consider as meaningful binary */
const MIN_DECODED_SIZE = 128;
/** MIME types that are inherently binary (skip looksLikeBinary verification on decoded bytes) */
const BINARY_MIME_PREFIXES = ['image/', 'audio/', 'video/', 'application/pdf', 'application/zip', 'application/gzip', 'application/octet-stream'];
/** Data URL regex: data:<mime>;base64,<payload> */
const DATA_URL_RE = /^data:([^;,]+);base64,(.+)$/s;
/**
* Result of extracting base64-encoded binary from a string.
*/
export interface Base64ExtractionResult {
buffer: Buffer;
mimeType: string | null;
/** File extension (with dot) derived from MIME or magic bytes */
ext: string;
source: 'data-url' | 'raw-base64';
}
/**
* Check if a MIME type is inherently binary (no need to verify decoded bytes).
*/
function isBinaryMime(mime: string): boolean {
const normalized = mime.toLowerCase().trim();
return BINARY_MIME_PREFIXES.some(prefix => normalized.startsWith(prefix));
}
/**
* Try to extract base64-encoded binary content from a text string.
*
* Handles two forms:
* 1. Data URLs: `data:<mime>;base64,<payload>`
* 2. Raw base64 blobs: long strings of base64 characters
*
* Two-step verification to minimize false positives:
* - Charset + structure check (is it plausibly base64?)
* - Decode + binary verification (are the decoded bytes actually binary?)
*
* Returns null if the string doesn't contain extractable base64 binary.
*/
export function extractBase64Binary(text: string): Base64ExtractionResult | null {
const trimmed = text.trim();
// --- Path A: Data URL ---
const dataUrlMatch = trimmed.match(DATA_URL_RE);
if (dataUrlMatch) {
const mime = dataUrlMatch[1]!;
const payload = dataUrlMatch[2]!;
if (payload.length < MIN_BASE64_LENGTH) return null;
try {
const decoded = Buffer.from(payload, 'base64');
if (decoded.length < MIN_DECODED_SIZE) return null;
// For known binary MIME types, trust the MIME — skip looksLikeBinary check
if (!isBinaryMime(mime) && !looksLikeBinary(decoded)) return null;
const ext = getMimeExtension(mime, decoded) || '.bin';
return { buffer: decoded, mimeType: mime, ext, source: 'data-url' };
} catch {
return null;
}
}
// --- Path B: Raw base64 blob ---
// Strict canonicalization pipeline — rejects anything that isn't structurally
// valid base64. Eliminates false positives from Node's lenient Buffer.from().
if (trimmed.length < MIN_BASE64_LENGTH) return null;
// Quick reject: structured data delimiters
const firstChar = trimmed.charCodeAt(0);
if (firstChar === 0x7B || firstChar === 0x5B || firstChar === 0x3C) return null; // { [ <
// Step 1: Strip only CR/LF (standard base64 line wrapping per RFC 2045).
// Spaces are NOT stripped — real base64 never contains spaces.
const stripped = trimmed.replace(/[\r\n]/g, '');
// Step 2: Strict charset — detect alphabet variant.
// Standard: [A-Za-z0-9+/] with optional = padding
// URL-safe: [A-Za-z0-9\-_] with optional = padding
const isStandard = /^[A-Za-z0-9+/]+=*$/.test(stripped);
const isUrlSafe = !isStandard && /^[A-Za-z0-9\-_]+=*$/.test(stripped);
if (!isStandard && !isUrlSafe) return null;
// Step 3: Normalize to standard alphabet for decoding
const normalized = isUrlSafe
? stripped.replace(/-/g, '+').replace(/_/g, '/')
: stripped;
// Step 4: Auto-pad to make length divisible by 4
const padded = normalized.length % 4 === 0
? normalized
: normalized + '='.repeat((4 - (normalized.length % 4)) % 4);
// Step 5: Decode
let decoded: Buffer;
try {
decoded = Buffer.from(padded, 'base64');
} catch {
return null;
}
if (decoded.length < MIN_DECODED_SIZE) return null;
// Step 6: Canonical roundtrip — re-encode and compare to padded input.
// Catches any input that Node's lenient decoder silently mangled.
if (decoded.toString('base64') !== padded) return null;
// Step 7: Binary-likeness check (unchanged)
if (!looksLikeBinary(decoded)) return null;
const ext = detectExtensionFromMagic(decoded) || '.bin';
return { buffer: decoded, mimeType: null, ext, source: 'raw-base64' };
}
// ============================================================
// File Saving
// ============================================================
/** Format bytes to human-readable string. */
export function formatBytes(bytes: number): string {
if (bytes === 0) return '0 B';
const k = 1024;
const sizes = ['B', 'KB', 'MB', 'GB'];
const i = Math.floor(Math.log(bytes) / Math.log(k));
const size = bytes / Math.pow(k, i);
return `${size.toFixed(i > 0 ? 1 : 0)} ${sizes[i]}`;
}
/** Sanitize filename by removing unsafe characters. */
export function sanitizeFilename(filename: string): string {
return filename
.replace(/[/\\:*?"<>|]/g, '_')
.replace(/\s+/g, '_')
.replace(/_+/g, '_')
.replace(/^_|_$/g, '')
.slice(0, 200);
}
/**
* Binary download result returned to the agent.
*/
export interface BinaryDownloadResult {
type: 'file_download';
path: string;
filename: string;
mimeType: string | null;
size: number;
sizeHuman: string;
}
/**
* Error result returned when binary save fails.
*/
export interface BinaryDownloadError {
type: 'file_download_error';
error: string;
}
/**
* Save binary response to session's downloads folder.
* Uses atomic file creation (O_EXCL) to prevent TOCTOU race conditions.
*/
export function saveBinaryResponse(
sessionPath: string,
filename: string,
buffer: Buffer,
mimeType: string | null
): BinaryDownloadResult | BinaryDownloadError {
const downloadsDir = join(sessionPath, 'downloads');
try {
mkdirSync(downloadsDir, { recursive: true });
} catch (err) {
return {
type: 'file_download_error',
error: `Failed to create downloads directory: ${(err as Error).message}`,
};
}
let finalFilename = filename;
let filePath = join(downloadsDir, finalFilename);
let counter = 0;
const maxAttempts = 100;
while (counter < maxAttempts) {
try {
writeFileSync(filePath, buffer, { flag: 'wx' });
return {
type: 'file_download',
path: filePath,
filename: finalFilename,
mimeType,
size: buffer.length,
sizeHuman: formatBytes(buffer.length),
};
} catch (err: unknown) {
const error = err as NodeJS.ErrnoException;
if (error.code === 'EEXIST') {
counter++;
const dotIdx = filename.lastIndexOf('.');
if (dotIdx > 0) {
const base = filename.slice(0, dotIdx);
const ext = filename.slice(dotIdx);
finalFilename = `${base}-${counter}${ext}`;
} else {
finalFilename = `${filename}-${counter}`;
}
filePath = join(downloadsDir, finalFilename);
} else {
return {
type: 'file_download_error',
error: `Failed to save file: ${error.message}`,
};
}
}
}
return {
type: 'file_download_error',
error: `Failed to save file after ${maxAttempts} attempts - too many collisions`,
};
}