-
Notifications
You must be signed in to change notification settings - Fork 3.7k
Expand file tree
/
Copy pathknowledge-processing.ts
More file actions
70 lines (63 loc) · 2.28 KB
/
Copy pathknowledge-processing.ts
File metadata and controls
70 lines (63 loc) · 2.28 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
import { createLogger } from '@sim/logger'
import { task } from '@trigger.dev/sdk'
import { env, envNumber } from '@/lib/core/config/env'
import {
assertDocumentProcessingPayload,
type DocumentProcessingBillingContext,
type DocumentProcessingPayload,
} from '@/lib/knowledge/documents/processing-payload'
import { processDocumentAsync } from '@/lib/knowledge/documents/service'
const logger = createLogger('TriggerKnowledgeProcessing')
export async function runDocumentProcessing(rawPayload: DocumentProcessingPayload) {
const payload = assertDocumentProcessingPayload(rawPayload)
const { knowledgeBaseId, documentId, docData, processingOptions, requestId } = payload
const billingContext: DocumentProcessingBillingContext =
payload.billingScope === 'workspace'
? {
billingScope: 'workspace',
actorUserId: payload.actorUserId,
workspaceId: payload.workspaceId,
billingAttribution: payload.billingAttribution,
}
: {
billingScope: 'non-workspace',
actorUserId: payload.actorUserId,
workspaceId: null,
}
logger.info(`[${requestId}] Starting Trigger.dev processing for document: ${docData.filename}`)
try {
await processDocumentAsync(
knowledgeBaseId,
documentId,
docData,
processingOptions,
billingContext
)
logger.info(`[${requestId}] Successfully processed document: ${docData.filename}`)
return {
success: true,
documentId,
filename: docData.filename,
processingTime: Date.now(),
}
} catch (error) {
logger.error(`[${requestId}] Failed to process document: ${docData.filename}`, error)
throw error
}
}
export const processDocument = task({
id: 'knowledge-process-document',
maxDuration: envNumber(env.KB_CONFIG_MAX_DURATION, 600),
machine: 'large-1x', // 2 vCPU, 2GB RAM - needed for large PDF processing
retry: {
maxAttempts: envNumber(env.KB_CONFIG_MAX_ATTEMPTS, 3),
factor: envNumber(env.KB_CONFIG_RETRY_FACTOR, 2),
minTimeoutInMs: envNumber(env.KB_CONFIG_MIN_TIMEOUT, 1000),
maxTimeoutInMs: envNumber(env.KB_CONFIG_MAX_TIMEOUT, 10000),
},
queue: {
concurrencyLimit: envNumber(env.KB_CONFIG_CONCURRENCY_LIMIT, 20),
name: 'document-processing-queue',
},
run: runDocumentProcessing,
})