-
Notifications
You must be signed in to change notification settings - Fork 2
Expand file tree
/
Copy pathcli.ts
More file actions
1653 lines (1484 loc) · 59.8 KB
/
Copy pathcli.ts
File metadata and controls
1653 lines (1484 loc) · 59.8 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
import type {
BackfillCandidate,
BackfillPlan,
BackfillSourceId,
CanonicalEvent,
SessionRollup,
} from '@codetime/shared'
import type { BackfillSourceDefinition } from './lib/backfill.js'
import type { BackfillImportCounts, BackfillIncrementalState, BackfillSourceFile, ParsedArgs, RunContext, SyncLocalLock, SyncLocalTriggerState, WritableLike } from './lib/types.js'
import { spawn } from 'node:child_process'
import { realpath, rm, stat } from 'node:fs/promises'
import os from 'node:os'
import path from 'node:path'
import { fileURLToPath } from 'node:url'
import {
AGENT_TIME_SCHEMA_VERSION,
BACKFILL_SOURCE_IDS,
createImportKey,
createPayloadHash,
createStableHash,
} from '@codetime/shared'
import { cac } from 'cac'
import { ampBackfillFiles, createAmpAdapter } from './adapters/amp.js'
import { createClaudeCodeAdapter } from './adapters/claude-code.js'
import { codexBackfillFiles, createCodexAdapter } from './adapters/codex.js'
import { createGeminiAdapter, geminiBackfillFiles } from './adapters/gemini.js'
import { createKimiAdapter, kimiBackfillFiles } from './adapters/kimi.js'
import { createOpenCodeAdapter, opencodeBackfillFiles } from './adapters/opencode.js'
import { createPiAdapter } from './adapters/pi.js'
import { AdapterRegistry } from './adapters/registry.js'
import { buildSessionRollups } from './backfill/rollup.js'
import { installEntry } from './install/manager.js'
import { matchesBackfillFilters } from './lib/backfill.js'
import { defaultMachineName, ensureLocalMachineId, readConfig, writeConfig } from './lib/config.js'
import { DEFAULT_API_URL, DEFAULT_BACKFILL_BATCH_BYTES, DEFAULT_BACKFILL_BATCH_SIZE, DEFAULT_HOOK_SYNC_MIN_INTERVAL_SECONDS, PACKAGE_VERSION } from './lib/constants.js'
import { isPlainObject, numberOption, stringOption, valuesOption } from './lib/fields.js'
import { countDirectoryEntries, listJsonlFiles, pathExists, readJsonIfExistsTolerant, writeFileAtomic } from './lib/fs.js'
import { logError } from './lib/logger.js'
import { isHeadless, openBrowser, sleep } from './lib/login.js'
import { ProgressBar } from './lib/progress.js'
import {
deleteMachine,
deleteRollupsBySource,
listMachines,
pollCliLink,
postRollupBatch,
renameMachine,
resolveRemote,
startCliLink,
} from './lib/remote.js'
import { BACKFILL_STATE_SCHEMA_VERSION } from './lib/types.js'
// ── Registry ──
function createRegistry(): AdapterRegistry {
const registry = new AdapterRegistry()
registry.register(createCodexAdapter())
registry.register(createClaudeCodeAdapter())
registry.register(createPiAdapter())
registry.register(createOpenCodeAdapter())
registry.register(createAmpAdapter())
registry.register(createGeminiAdapter())
registry.register(createKimiAdapter())
return registry
}
// ── Run context ──
const defaultContext: RunContext = {
env: process.env,
stdin: process.stdin,
stdout: process.stdout,
stderr: process.stderr,
fetch: globalThis.fetch,
spawn,
}
export async function run(argv: string[], context: Partial<RunContext> = {}): Promise<number> {
const ctx = { ...defaultContext, ...context }
const cli = createCli(ctx, createRegistry())
try {
if (argv.length === 0) {
write(ctx.stdout, helpText())
return 0
}
cli.parse(['node', 'codetime', ...argv], { run: false })
if (cli.options.help) {
write(ctx.stdout, helpText())
return 0
}
if (cli.options.version && !cli.matchedCommandName) {
write(ctx.stdout, `${PACKAGE_VERSION}\n`)
return 0
}
if (!cli.matchedCommand) {
const command = cli.args[0]
write(ctx.stderr, `Unknown command: ${command}\n\n${helpText()}`)
return 1
}
return Number(await cli.runMatchedCommand()) || 0
}
catch (error) {
write(ctx.stderr, `${(error as Error).message}\n`)
await logError('cli', error, { argv })
return 1
}
}
// ── CLI Definition ──
function createCli(ctx: RunContext, registry: AdapterRegistry) {
const cli = cac('codetime')
cli
.option('-h, --help', 'Show help')
.option('-v, --version', 'Print CLI version')
.option('--home <path>', 'Override the user home directory')
.option('--api-url <url>', 'Agent Time API URL')
.option('--token <token>', 'Bearer token for the Agent Time API')
.option('--dry-run', 'Print the planned action without writing or reporting')
.option('--json', 'Print JSON output')
cli.command('help', 'Show help')
.action(() => {
write(ctx.stdout, helpText()); return 0
})
cli.command('version', 'Print CLI version')
.action(() => {
write(ctx.stdout, `${PACKAGE_VERSION}\n`); return 0
})
cli.command('detect', 'Show supported local targets and install status')
.action(options => detectCommand(normalizeOptions(options), ctx, registry).then(() => 0))
cli.command('install', 'Install integration files into detected or requested targets')
.option('--target <targets>', 'Target integrations, comma-separated')
.option('--targets <targets>', 'Target integrations, comma-separated')
.option('--all', 'Install all supported integrations')
.option('--force', 'Overwrite existing non-generated files when needed')
.action(options => installCommand(normalizeOptions(options), ctx, registry))
cli.command('hook', 'Read agent hook JSON from stdin and report a throttled event')
.option('--agent <name>', 'Agent name')
.option('--project <name>', 'Project name')
.option('--min-interval <seconds>', 'Minimum seconds between similar hook reports')
.action(options => hookCommand(normalizeOptions(options), ctx))
cli.command('sync-local-trigger', 'Trigger one background local sync with throttle and locking')
.option('--min-interval <seconds>', 'Minimum seconds between sync triggers')
.action(options => syncLocalTriggerCommand(normalizeOptions(options), ctx, registry))
cli.command('sync-local-runner', 'Internal background local sync runner')
.option('--lock-file <path>', 'Lock file for the active sync')
.option('--state-file <path>', 'State file for trigger metadata')
.action(options => syncLocalRunnerCommand(normalizeOptions(options), ctx, registry))
cli.command('backfill [action]', 'Inspect local history import candidates')
.option('--source <source>', 'Backfill source')
.option('--since <time>', 'Only include history after this time')
.option('--until <time>', 'Only include history before this time')
.option('--project <name>', 'Project filter')
.option('--source-root <path>', 'Override source history root')
.option('--include-source-path', 'Include local source paths in output')
.option('--import-run <id>', 'Import run id for verify/resume workflows')
.option('--limit <count>', 'Maximum session files to parse')
.option('--batch-size <count>', 'Max rollups per request (also bounded by --batch-bytes)')
.option('--batch-bytes <bytes>', 'Soft byte cap for the JSON body of a single ingest POST')
.option('--replace', 'Replace conflicting records during import (default)')
.option('--skip-conflicts', 'Skip conflicting records instead of replacing them')
.option('--force', 'Full re-import: re-parse every file and overwrite matching rollups (non-destructive — nothing is deleted)')
.option('--purge', 'Before re-importing, delete THIS machine\'s existing rollups for the source(s). Destructive: also removes rollups whose local files are gone. Implies --force')
.action((action, options) => backfillCommand({ ...normalizeOptions(options), action }, ctx, registry))
// `sync` is the friendly front door to the upload path: it just runs
// `backfill import --source all`. Most users want "send my local
// history now", and the bare `backfill` defaults to a dry plan — this
// removes that surprise without changing backfill's own semantics.
cli.command('sync', 'Import and upload all local agent history (shorthand for `backfill import --source all`)')
.option('--source <source>', 'Limit to one source (default: all)')
.option('--since <time>', 'Only include history after this time')
.option('--until <time>', 'Only include history before this time')
.option('--project <name>', 'Project filter')
.option('--batch-size <count>', 'Max rollups per request (also bounded by --batch-bytes)')
.option('--force', 'Full re-import: re-parse every file and overwrite matching rollups (non-destructive — nothing is deleted)')
.option('--purge', 'Before re-importing, delete THIS machine\'s existing rollups for the source(s). Destructive: also removes rollups whose local files are gone. Implies --force')
.option('--dry-run', 'Print the planned import without uploading')
.action((options) => {
const opts = normalizeOptions(options)
return backfillCommand({ ...opts, action: 'import', source: stringOption(opts.source) || 'all' }, ctx, registry)
})
// Browser login (device-code flow): opens `<remote>/cli/auth?code=…`,
// polls until the user approves it there, then writes the upload token
// to config — the one-click alternative to `token set`. Works over SSH
// too: open the printed URL on any device. See lib/login.ts + remote.ts.
cli.command('login', 'Authorize this machine by signing in through your browser')
.option('--remote <url>', 'Override API base URL for this login')
.option('--no-browser', 'Print the login URL instead of opening a browser')
.action(options => loginCommand(normalizeOptions(options), ctx))
// `token` is the manual alternative to `login`: the agent CLI reuses
// the user's existing upload_token (visible in the codetime
// dashboard's Settings page).
// token set <value> write to ~/.codetime/config.json
// token show print masked token + remoteUrl
// token clear remove only the token (keep remoteUrl)
cli.command('token [action] [value]', 'Set, show, or clear the persisted API token')
.option('--remote <url>', 'Override API base URL when setting a token')
.action((action, value, options) => tokenCommand(action, value, normalizeOptions(options), ctx))
cli.command('machine [action]', 'List or rename machines (requires login)')
.option('--name <name>', 'New display name (used by `machine rename`)')
.option('--id <id>', 'Machine id (defaults to current machine)')
.action((action, options) => machineCommand(action, normalizeOptions(options), ctx))
return cli
}
function normalizeOptions(options: Record<string, unknown>): ParsedArgs {
const normalized: ParsedArgs = { ...options, _: [] }
const aliases: Record<string, string> = {
apiUrl: 'api-url',
dryRun: 'dry-run',
linesAdded: 'lines-added',
linesRemoved: 'lines-removed',
minInterval: 'min-interval',
lockFile: 'lock-file',
stateFile: 'state-file',
sourceRoot: 'source-root',
importRun: 'import-run',
batchSize: 'batch-size',
batchBytes: 'batch-bytes',
skipConflicts: 'skip-conflicts',
}
for (const [camel, dashed] of Object.entries(aliases)) {
if (normalized[camel] !== undefined && normalized[dashed] === undefined) {
normalized[dashed] = normalized[camel]
}
}
return normalized
}
// ── Commands ──
async function detectCommand(options: ParsedArgs, ctx: RunContext, registry: AdapterRegistry) {
const home = resolveHome(options, ctx)
const env = ctx.env
const adapters = registry.all()
const targets = await Promise.all(adapters.map(async (adapter) => {
const detected = await pathExists(adapter.detectPath(home, env))
const installed = await adapter.isInstalled(home, env)
return {
id: adapter.id,
label: adapter.label,
kind: adapter.kind,
detected,
installed,
detectPath: adapter.detectPath(home, env),
installedPath: adapter.installedPath(home, env),
}
}))
if (options.json) {
write(ctx.stdout, `${JSON.stringify({ home, targets }, null, 2)}\n`)
return
}
for (const target of targets) {
const detected = target.detected ? 'detected' : 'missing'
const installed = target.installed ? 'installed' : 'not installed'
write(ctx.stdout, `${target.id.padEnd(8)} ${detected.padEnd(8)} ${installed.padEnd(13)} ${target.detectPath}\n`)
}
}
async function installCommand(options: ParsedArgs, ctx: RunContext, registry: AdapterRegistry): Promise<number> {
const home = resolveHome(options, ctx)
const env = ctx.env
const dryRun = Boolean(options['dry-run'])
const force = Boolean(options.force)
const allAdapters = registry.all()
const requested = requestedTargets(options)
const unknown = requested.filter(id => !allAdapters.some(a => a.id === id))
if (unknown.length > 0) {
throw new Error(`Unknown target(s): ${unknown.join(', ')}`)
}
const detected: string[] = []
for (const adapter of allAdapters) {
if (await pathExists(adapter.detectPath(home, env))) {
detected.push(adapter.id)
}
}
const selectedIds = requested.length > 0
? requested
: options.all
? allAdapters.map(a => a.id)
: detected
if (selectedIds.length === 0) {
write(ctx.stderr, 'No supported local targets were detected. Use --target codex,claude,opencode,pi,amp or --all to create them.\n')
return 1
}
for (const adapter of allAdapters.filter(a => selectedIds.includes(a.id))) {
for (const entry of adapter.installEntries(home, env)) {
await installEntry(entry, {
dryRun,
force,
onWrite: msg => write(ctx.stdout, `${msg}\n`),
})
}
}
return 0
}
// The hook command is a thin trigger: it drains stdin so the upstream agent
// doesn't block on a closed pipe, then schedules a local backfill run.
// Backfill's mtime watermark and per-adapter parsers do all the real work
// (model.usage assembly, token dedup, service_tier rewrites, etc.) — keeping
// the hook side reactive but stateless avoids two copies of every parser.
async function hookCommand(options: ParsedArgs, ctx: RunContext): Promise<number> {
const home = resolveHome(options, ctx)
try {
const agent = requiredOption(options, 'agent')
const payload = await readHookPayload(ctx.stdin)
if (options['dry-run']) {
// Echo the raw payload so users debugging hook wiring can see exactly
// what the agent forwarded. No event assembly, no cost estimate —
// those happen on the backfill side.
write(ctx.stdout, `${JSON.stringify({
agent,
received: payload,
wouldTrigger: 'backfill',
}, null, 2)}\n`)
return 0
}
return await syncLocalTriggerCommand({
...options,
agent,
'min-interval': stringOption(options['min-interval']) || String(DEFAULT_HOOK_SYNC_MIN_INTERVAL_SECONDS),
}, ctx)
}
catch (error) {
// Hooks run inside the user's agent (Claude Code, Codex, etc).
// Bubbling an error there spams the user with stderr; persist to the
// log file and exit 0 so the agent isn't disturbed.
await logError('hook', error, { agent: stringOption(options.agent) }, home)
debug(ctx, `[codetime] hook failed: ${(error as Error).message}\n`)
return 0
}
}
async function syncLocalTriggerCommand(options: ParsedArgs, ctx: RunContext, _registry?: AdapterRegistry): Promise<number> {
const home = resolveHome(options, ctx)
const statePath = syncLocalTriggerStatePath(home)
const lockPath = syncLocalTriggerLockPath(home)
const minIntervalSeconds = Math.max(0, Math.floor(numberOption(options['min-interval']) ?? DEFAULT_HOOK_SYNC_MIN_INTERVAL_SECONDS))
const now = new Date().toISOString()
const lock = await readSyncLocalLock(lockPath)
if (lock && await isProcessRunning(lock.pid)) {
if (options.json || options['dry-run']) {
write(ctx.stdout, `${JSON.stringify({ status: 'already-running', pid: lock.pid, startedAt: lock.startedAt }, null, 2)}\n`)
}
return 0
}
if (lock) {
await clearSyncLocalLock(lockPath)
}
const state = await readSyncLocalTriggerState(statePath)
if (minIntervalSeconds > 0 && state.lastTriggeredAt) {
const elapsedMs = Date.parse(now) - Date.parse(state.lastTriggeredAt)
if (Number.isFinite(elapsedMs) && elapsedMs >= 0 && elapsedMs < minIntervalSeconds * 1000) {
if (options.json || options['dry-run']) {
write(ctx.stdout, `${JSON.stringify({ status: 'throttled', lastTriggeredAt: state.lastTriggeredAt, minIntervalSeconds }, null, 2)}\n`)
}
return 0
}
}
if (options['dry-run']) {
write(ctx.stdout, `${JSON.stringify({ status: 'would-trigger', minIntervalSeconds }, null, 2)}\n`)
return 0
}
const child = spawnSyncLocalRunner({ options, ctx, home, lockPath, statePath, triggeredAt: now })
if (typeof child.pid !== 'number') {
throw new TypeError('Could not start background sync-local runner')
}
state.lastTriggeredAt = now
state.pid = child.pid
await writeSyncLocalTriggerState(statePath, state)
await writeSyncLocalLock(lockPath, { pid: child.pid, startedAt: now })
return 0
}
async function syncLocalRunnerCommand(options: ParsedArgs, ctx: RunContext, _registry?: AdapterRegistry): Promise<number> {
const home = resolveHome(options, ctx)
const lockPath = stringOption(options['lock-file']) || syncLocalTriggerLockPath(home)
const statePath = stringOption(options['state-file']) || syncLocalTriggerStatePath(home)
const state = await readSyncLocalTriggerState(statePath)
state.lastStartedAt = new Date().toISOString()
state.pid = process.pid
await writeSyncLocalTriggerState(statePath, state)
let exitCode = 1
try {
exitCode = await backfillCommand({
...options,
action: 'import',
source: 'all',
}, ctx)
return exitCode
}
catch (error) {
// The runner is spawned detached with stdio: 'ignore', so the stack
// trace would be lost otherwise. Persist it so users can diagnose.
await logError('sync-local-runner', error, { home }, home)
throw error
}
finally {
const nextState = await readSyncLocalTriggerState(statePath)
nextState.lastStartedAt = state.lastStartedAt
nextState.lastCompletedAt = new Date().toISOString()
nextState.lastExitCode = exitCode
delete nextState.pid
await writeSyncLocalTriggerState(statePath, nextState)
await clearSyncLocalLock(lockPath)
}
}
// ── Backfill ──
async function backfillCommand(options: ParsedArgs, ctx: RunContext, registry?: AdapterRegistry): Promise<number> {
const reg = registry || createRegistry()
const action = stringOption(options.action) || 'plan'
if (!['discover', 'plan', 'import', 'verify'].includes(action)) {
throw new Error(`Unknown backfill action: ${action}`)
}
if (action === 'verify') {
return backfillVerifyCommand(options, ctx)
}
if (action === 'import' && !options['dry-run']) {
const requested = normalizeBackfillSource(stringOption(options.source) || 'all')
const supported = new Set<string>(['all', ...BACKFILL_SOURCE_IDS])
if (!supported.has(requested)) {
write(ctx.stderr, `Unsupported backfill source: ${requested}\n`)
return 1
}
const plan = await createBackfillPlanFromOptions(options, ctx, 'discover', reg)
return importBackfillPlan(plan, options, ctx, reg)
}
const plan = await createBackfillPlanFromOptions(options, ctx, action, reg)
if (action === 'discover') {
writeBackfillDiscover(plan, options, ctx)
return 0
}
if (action === 'plan' || options['dry-run']) {
writeBackfillPlan(plan, options, ctx)
return 0
}
return importBackfillPlan(plan, options, ctx, reg)
}
async function createBackfillPlanFromOptions(
options: ParsedArgs,
ctx: RunContext,
action: string,
registry: AdapterRegistry,
): Promise<BackfillPlan> {
const home = resolveHome(options, ctx)
const env = ctx.env
const source = normalizeBackfillSource(stringOption(options.source) || 'all')
const sourceDefs = source === 'all'
? registry.all().map(a => ({ id: a.id, label: a.label, paths: a.sourcePaths(home, env) }))
: (() => {
const adapter = registry.get(source)
if (!adapter) {
return []
}
return [{ id: adapter.id, label: adapter.label, paths: adapter.sourcePaths(home, env) }]
})()
if (sourceDefs.length === 0) {
throw new Error(`Unknown backfill source: ${source}`)
}
const candidateList = await Promise.all(sourceDefs.map(item => createBackfillCandidates(item, options)))
const candidates = candidateList.flat()
let events: CanonicalEvent[] = []
if (action !== 'discover') {
const eventList = await createBackfillEventsFromDefs(sourceDefs, options, registry, ctx)
events = eventList.flat()
}
const plannedEvents = events.map(event => ({
source: event.source as BackfillSourceId,
importKey: event.refs?.importKey || event.id || '',
eventId: event.id || '',
payloadHash: createPayloadHash(event),
type: event.type,
confidence: event.confidence || 'estimated',
}))
const now = new Date().toISOString()
const importRunId = `import_${createStableHash(createImportKey([
source,
action,
now,
stringOption(options.since),
stringOption(options.until),
stringOption(options.project),
stringOption(options['source-root']),
])).slice(0, 24)}`
return {
importRun: {
importRunId,
source: source === 'all' ? 'all' : source as BackfillSourceId,
status: 'planned',
startedAt: now,
parserVersion: PACKAGE_VERSION,
schemaVersion: AGENT_TIME_SCHEMA_VERSION,
dryRun: Boolean(options['dry-run']) || action !== 'import',
filters: {
since: stringOption(options.since),
until: stringOption(options.until),
project: stringOption(options.project),
sourceRoot: stringOption(options['source-root']),
},
counts: {
discovered: candidates.reduce((total, c) => total + (c.exists ? c.entries : 0), 0),
planned: plannedEvents.length,
inserted: 0,
skipped: 0,
conflicts: 0,
failed: 0,
},
},
candidates,
plannedEvents,
privacy: 'metadata only; prompt text, command text, source code, and diffs are not imported',
}
}
async function createBackfillEventsFromDefs(
sourceDefs: BackfillSourceDefinition[],
options: ParsedArgs,
registry: AdapterRegistry,
ctx: RunContext,
overrideFiles?: string[],
): Promise<CanonicalEvent[]> {
const events: CanonicalEvent[] = []
const home = resolveHome(options, ctx)
// Same isolation policy as the import path: one source blowing up
// (e.g. older opencode SQLite schemas) must not poison the whole plan.
for (const item of sourceDefs) {
const parser = registry.getParser(item.id)
if (!parser) {
continue
}
let files: string[]
try {
const sourceFiles = await listBackfillSourceFiles(item, options, ctx)
files = overrideFiles ?? sourceFiles.map(f => f.path)
}
catch (error) {
await logError('backfill.listFiles', error, { source: item.id, phase: 'plan' }, home)
debug(ctx, `[codetime] skip ${item.id} in plan: list files failed: ${(error as Error).message}\n`)
continue
}
for (const filePath of files) {
try {
const parsed = await parser(filePath, options)
for (const event of parsed) {
if (matchesBackfillFilters(event, options)) {
events.push(event)
}
}
}
catch (error) {
await logError('backfill.parse', error, { source: item.id, file: filePath, phase: 'plan' }, home)
debug(ctx, `[codetime] skip ${item.id} file ${filePath} in plan: ${(error as Error).message}\n`)
}
}
}
return events
}
async function createBackfillCandidates(
source: BackfillSourceDefinition,
options: ParsedArgs,
): Promise<BackfillCandidate[]> {
const sourceRoot = stringOption(options['source-root'])
const paths = sourceRoot ? [sourceRoot] : source.paths
return Promise.all(paths.map(async (candidatePath) => {
const exists = await pathExists(candidatePath)
return {
source: source.id,
label: source.label,
exists,
entries: exists ? await countDirectoryEntries(candidatePath) : 0,
pathHash: `sha256:${createStableHash(candidatePath)}`,
path: options.includeSourcePath ? candidatePath : undefined,
}
}))
}
async function listBackfillSourceFiles(
source: BackfillSourceDefinition,
options: ParsedArgs,
ctx: RunContext,
): Promise<BackfillSourceFile[]> {
if (source.id === 'opencode') {
return canonicalizeBackfillFiles(await opencodeBackfillFiles(stringOption(options['source-root']), resolveHome(options, ctx), ctx.env))
}
if (source.id === 'amp') {
return canonicalizeBackfillFiles(await ampBackfillFiles(stringOption(options['source-root']), resolveHome(options, ctx), ctx.env))
}
if (source.id === 'gemini') {
return canonicalizeBackfillFiles(await geminiBackfillFiles(stringOption(options['source-root']), resolveHome(options, ctx), ctx.env))
}
// Kimi keeps non-usage jsonl next to each run's wire.jsonl and nests it at two
// different depths, so the generic "every .jsonl under the root" listing would
// both miss and over-collect.
if (source.id === 'kimi') {
return canonicalizeBackfillFiles(await kimiBackfillFiles(stringOption(options['source-root']), resolveHome(options, ctx), ctx.env))
}
if (source.id === 'codex') {
const files = await codexBackfillFiles(stringOption(options['source-root']), resolveHome(options, ctx), ctx.env)
return canonicalizeBackfillFiles(files
.sort((a, b) => a.path.localeCompare(b.path))
.slice(0, numberOption(options.limit) || undefined))
}
const roots = stringOption(options['source-root'])
? [requiredOption(options, 'source-root')]
: source.paths
const fileLists = await Promise.all(roots.map(r => listJsonlFiles(r)))
const files = fileLists
.flat()
.sort()
.slice(0, numberOption(options.limit) || undefined)
return canonicalizeBackfillFiles(await Promise.all(files.map(async (filePath) => {
const info = await stat(filePath)
return { path: filePath, modifiedAt: info.mtime.toISOString() }
})))
}
// Rollup identity includes a hash of the file's absolute path, so the same
// physical file reached through different paths (symlinked agent homes like
// CODEX_HOME=~/.codex-work → ~/.codex, macOS /var → /private/var) would
// otherwise upload duplicate rollups — a hook-triggered sync and a manual one
// can legitimately see different spellings of the same home. Resolve every
// listed file to its physical path before parsing so all views agree on one
// identity, and drop same-run duplicates that collapse together.
async function canonicalizeBackfillFiles(files: BackfillSourceFile[]): Promise<BackfillSourceFile[]> {
const seen = new Set<string>()
const result: BackfillSourceFile[] = []
for (const file of files) {
let resolved = file.path
try {
resolved = await realpath(file.path)
}
catch {
// Keep the literal path: a file that vanished between listing and now
// (or an unreadable parent) still fails later with proper per-file logging.
}
if (seen.has(resolved)) {
continue
}
seen.add(resolved)
result.push(resolved === file.path ? file : { ...file, path: resolved })
}
return result
}
function writeBackfillDiscover(plan: BackfillPlan, options: ParsedArgs, ctx: RunContext) {
if (options.json) {
write(ctx.stdout, `${JSON.stringify({ importRun: plan.importRun, candidates: plan.candidates }, null, 2)}\n`)
return
}
for (const candidate of plan.candidates) {
const state = candidate.exists ? `found ${candidate.entries} entries` : 'missing'
write(ctx.stdout, `${candidate.source.padEnd(12)} ${state.padEnd(16)} ${candidate.path || candidate.pathHash}\n`)
}
}
function formatCountMap(map: Map<string, number>, limit = 6): string {
const entries = [...map.entries()].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))
const visible = entries.slice(0, limit).map(([key, count]) => `${key}:${count}`)
const rest = entries.slice(limit).reduce((total, [, count]) => total + count, 0)
return rest > 0 ? `${visible.join(', ')} (+${rest} more)` : visible.join(', ')
}
function writeBackfillPlan(plan: BackfillPlan, options: ParsedArgs, ctx: RunContext) {
if (options.json) {
write(ctx.stdout, `${JSON.stringify(plan, null, 2)}\n`)
return
}
write(ctx.stdout, `importRun ${plan.importRun.importRunId}\n`)
write(ctx.stdout, `source ${plan.importRun.source}\n`)
write(ctx.stdout, `discovered ${plan.importRun.counts.discovered}\n`)
write(ctx.stdout, `planned ${plan.importRun.counts.planned}\n`)
write(ctx.stdout, `${plan.privacy}\n`)
const candidatesBySource = new Map<string, number>()
for (const candidate of plan.candidates) {
candidatesBySource.set(candidate.source, (candidatesBySource.get(candidate.source) || 0) + (candidate.exists ? candidate.entries : 0))
}
if (candidatesBySource.size > 0) {
write(ctx.stdout, `candidates ${formatCountMap(candidatesBySource)}\n`)
}
const eventsBySource = new Map<string, number>()
const eventsByType = new Map<string, number>()
for (const event of plan.plannedEvents) {
eventsBySource.set(event.source, (eventsBySource.get(event.source) || 0) + 1)
eventsByType.set(event.type, (eventsByType.get(event.type) || 0) + 1)
}
if (eventsBySource.size > 0) {
write(ctx.stdout, `events ${formatCountMap(eventsBySource)}\n`)
}
if (eventsByType.size > 0) {
write(ctx.stdout, `types ${formatCountMap(eventsByType, 8)}\n`)
}
const samples = plan.plannedEvents.slice(0, 5)
if (samples.length > 0) {
write(ctx.stdout, 'sample\n')
for (const event of samples) {
write(ctx.stdout, ` ${event.eventId} ${event.source} ${event.type} ${event.confidence}\n`)
}
const remaining = plan.plannedEvents.length - samples.length
if (remaining > 0) {
write(ctx.stdout, ` ... ${remaining} more planned events (use --json for full details)\n`)
}
}
}
async function importBackfillPlan(
plan: BackfillPlan,
options: ParsedArgs,
ctx: RunContext,
registry: AdapterRegistry,
): Promise<number> {
const source = normalizeBackfillSource(stringOption(options.source) || 'all')
const supportedSources = new Set<BackfillSourceId>(BACKFILL_SOURCE_IDS)
const home = resolveHome(options, ctx)
if (source !== 'all' && !supportedSources.has(source as BackfillSourceId)) {
write(ctx.stderr, `Unsupported backfill source: ${source}\n`)
return 1
}
const sourceDefs = registry.all()
.filter(a => supportedSources.has(a.id) && (source === 'all' || a.id === source))
.map(a => ({ id: a.id, label: a.label, paths: a.sourcePaths(home, ctx.env) }))
// --purge (opt-in, destructive): drop this machine's existing rollups for the
// source(s) BEFORE re-importing. Without it, --force is non-destructive: it
// re-parses every file and overwrites matching rollups via replace:true, so
// rollups for files that have since rolled off disk are preserved, not deleted.
if (options.purge) {
await purgeSourceRollups(sourceDefs, options, ctx)
}
// Both --force and --purge re-import everything: clear the watermark so no file
// is skipped as "already imported".
if (options.force || options.purge) {
await clearBackfillWatermark(home, ctx)
}
const incrementalState = shouldUseIncrementalBackfill(options)
? await readBackfillIncrementalState(home, ctx)
: undefined
if (!options.json) {
write(ctx.stdout, `importRun ${plan.importRun.importRunId}\n`)
write(ctx.stdout, `sources ${sourceDefs.map(s => s.id).join(', ') || 'none'}\n`)
}
const { canonicalEvents, selectedFilesBySource } = await collectCanonicalEvents(
sourceDefs,
registry,
incrementalState,
options,
ctx,
)
const rollups = buildSessionRollups(canonicalEvents)
const counts = await uploadSessionRollups(rollups, canonicalEvents.length, options, ctx)
const result = {
importRunId: plan.importRun.importRunId,
source,
planned: rollups.length,
sourceEvents: canonicalEvents.length,
...counts,
}
if (options.json) {
write(ctx.stdout, `${JSON.stringify(result, null, 2)}\n`)
}
if (counts.failed === 0 && counts.conflicts === 0 && incrementalState) {
await updateBackfillIncrementalState(home, incrementalState, selectedFilesBySource)
}
return counts.failed > 0 || (counts.conflicts > 0 && !options['skip-conflicts']) ? 1 : 0
}
// Non-destructive half of a full re-import: drop the local watermark so the next
// parse re-reads every file. Nothing is deleted server-side — re-parsed rollups
// overwrite their prior versions in place via replace:true, and rollups for files
// no longer on disk are simply left untouched (preserved).
async function clearBackfillWatermark(home: string, ctx: RunContext): Promise<void> {
try {
// rm({force: true}) already swallows ENOENT, so anything reaching
// here is a real I/O / permission problem.
await rm(backfillIncrementalStatePath(home), { force: true })
}
catch (error) {
debug(ctx, `Failed to clear backfill watermark: ${(error as Error).message}\n`)
}
}
// Destructive, opt-in via --purge: ask the server to drop this machine's existing
// rollups for each target source before reimporting. This removes rollups whose
// local files have rolled off disk, so use it only to clean up stale/incorrect
// data — routine re-imports should use --force. Errors are non-fatal; the import
// proceeds regardless.
async function purgeSourceRollups(
sourceDefs: Array<{ id: BackfillSourceId, label: string, paths: string[] }>,
options: ParsedArgs,
ctx: RunContext,
): Promise<void> {
for (const item of sourceDefs) {
try {
const deleted = await deleteSessionRollupsBySourceAPI(item.id, options, ctx)
if (!options.json) {
write(ctx.stdout, `purged ${item.id}: ${deleted} old rollups\n`)
}
}
catch (error) {
debug(ctx, `Failed to purge ${item.id} rollups: ${(error as Error).message}\n`)
}
}
}
// Walk each enabled source, run its parser over every file past the
// recorded watermark, and return the filtered canonical events. Also
// returns the per-source file lists so callers can advance watermarks
// once the upload succeeds.
async function collectCanonicalEvents(
sourceDefs: Array<{ id: BackfillSourceId, label: string, paths: string[] }>,
registry: AdapterRegistry,
incrementalState: BackfillIncrementalState | undefined,
options: ParsedArgs,
ctx: RunContext,
): Promise<{
canonicalEvents: CanonicalEvent[]
selectedFilesBySource: Map<BackfillSourceId, BackfillSourceFile[]>
}> {
const selectedFilesBySource = new Map<BackfillSourceId, BackfillSourceFile[]>()
const canonicalEvents: CanonicalEvent[] = []
const home = resolveHome(options, ctx)
for (const item of sourceDefs) {
const parser = registry.getParser(item.id)
if (!parser) {
continue
}
// Per-source isolation: a broken parser or missing history dir for
// one source (e.g. opencode's older schemas) must not abort the
// whole run. Failures land in ~/.codetime/logs/cli.log; the
// watermark stays unadvanced because selectedFilesBySource only
// gets populated on success.
let selectedFiles: BackfillSourceFile[]
try {
const sourceFiles = await listBackfillSourceFiles(item, options, ctx)
selectedFiles = selectBackfillFilesForImport(sourceFiles, incrementalState?.sources[item.id]?.watermarkTs)
}
catch (error) {
await logError('backfill.listFiles', error, { source: item.id }, home)
debug(ctx, `[codetime] skip ${item.id}: list files failed: ${(error as Error).message}\n`)
continue
}
selectedFilesBySource.set(item.id, selectedFiles)
const filePaths = selectedFiles.map(f => f.path)
const sourceEvents: CanonicalEvent[] = []
let sourceFailed = false
const bar = options.json ? undefined : new ProgressBar(ctx.stdout, `${item.id.padEnd(12)}`)
bar?.init(filePaths.length, `0 events`)
for (let fi = 0; fi < filePaths.length; fi += 1) {
try {
const parsed = await parser(filePaths[fi], options)
for (const event of parsed) {
if (matchesBackfillFilters(event, options)) {
sourceEvents.push(event)
}
}
}
catch (error) {
sourceFailed = true
await logError('backfill.parse', error, { source: item.id, file: filePaths[fi] }, home)
debug(ctx, `[codetime] skip ${item.id} file ${filePaths[fi]}: ${(error as Error).message}\n`)
}
bar?.tick(`${fi + 1}/${filePaths.length} files, ${sourceEvents.length} events`)
}
bar?.finalize(`${sourceEvents.length} events${sourceFailed ? ' (partial — see logs)' : ''}`)
// If any file failed to parse, drop this source from the watermark
// update set so we retry on the next run instead of marking it as
// fully imported.
if (sourceFailed) {
selectedFilesBySource.delete(item.id)
}
for (const event of sourceEvents) canonicalEvents.push(event)
}
return { canonicalEvents, selectedFilesBySource }
}
// Pre-pack rollups into batches bounded by BOTH count and serialized
// JSON byte size. The byte cap keeps us under nginx's default 1 MiB
// `client_max_body_size`; the count cap protects request latency on
// tiny rollups. A single rollup that exceeds the byte cap is sent on
// its own — the server may 413, surfaced as a batch failure.
async function uploadSessionRollups(
rollups: SessionRollup[],
eventCount: number,
options: ParsedArgs,
ctx: RunContext,
): Promise<BackfillImportCounts> {
const counts: BackfillImportCounts = { inserted: 0, skipped: 0, conflicts: 0, failed: 0 }
const batchSize = Math.max(1, Math.floor(numberOption(options['batch-size']) || DEFAULT_BACKFILL_BATCH_SIZE))
const batchBytes = Math.max(64 * 1024, Math.floor(numberOption(options['batch-bytes']) || DEFAULT_BACKFILL_BATCH_BYTES))
const batches = packRollupBatches(rollups, batchSize, batchBytes)
const totalBatches = batches.length
let uploadBar: ProgressBar | undefined
if (!options.json) {
write(ctx.stdout, `rollup ${rollups.length} from ${eventCount} events\n`)
uploadBar = new ProgressBar(ctx.stdout, `upload`.padEnd(12))
uploadBar.init(totalBatches, `0/${totalBatches} batches, 0 inserted`)
}
for (const [i, batch_] of batches.entries()) {
const batch = batch_!
const batchNumber = i + 1
try {
const result = await sendSessionRollupBatch(batch, options, ctx)
counts.inserted += result.inserted
counts.skipped += result.skipped
counts.conflicts += result.conflicts
counts.failed += result.failed
}
catch (error) {
debug(ctx, `backfill rollup batch ${batchNumber}/${totalBatches} (${batch.length} rollups) failed: ${(error as Error).message}\n`)
counts.failed += batch.length
}
uploadBar?.update(batchNumber, `${batchNumber}/${totalBatches} batches, inserted ${counts.inserted}`)
}
uploadBar?.finalize(`inserted ${counts.inserted} · skipped ${counts.skipped}${
counts.conflicts ? ` · conflicts ${counts.conflicts}` : ''
}${counts.failed ? ` · failed ${counts.failed}` : ''}`)
return counts
}
function backfillVerifyCommand(options: ParsedArgs, ctx: RunContext): number {