Repository navigation
Expand file tree
/
Copy pathinvocation.ts
More file actions
297 lines (272 loc) · 11.2 KB
/
Copy pathinvocation.ts
File metadata and controls
297 lines (272 loc) · 11.2 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
import type { Command } from 'commander'
import { cliVersion } from '#sim-cli/version'
import { profileFrom } from '../context'
import { isCi } from '../environment'
import { SimApiError } from '../http/client'
import { commandPath } from '../runtime/derive'
import { detectCodingAgent, NO_CODING_AGENT } from './coding-agent'
import { telemetryStatus } from './policy'
import { loadTelemetryState, nextSession, type TelemetryState, writeTelemetryState } from './state'
import {
builtInIngestTarget,
type CaptureRequest,
type IngestTarget,
sendCapture,
} from './transport'
/**
* Usage reporting for one CLI invocation: what ran, whether it worked, and
* how long it took. One event per command, sent after the command finishes.
*
* What is sent is the smallest set that answers "how is the CLI used": the
* command's name, the names of the flags typed, how many positional arguments
* there were, the exit code, the duration, and the runtime it ran on. What is
* never sent is anything the user typed — no argument values, flag values,
* paths, or error messages — matching the policy the Stripe, GitHub, and
* Supabase CLIs converge on. See {@link CommandEventProperties}.
*/
export const COMMAND_EVENT = 'cli_command_executed'
/** The user-facing name of this reporting, as `$lib` in each event. */
const LIBRARY_NAME = 'sim-cli'
/** The command group that manages reporting is never itself reported. */
const EXCLUDED_ROOT_COMMAND = 'telemetry'
export const USAGE_DATA_DOCS_URL = 'https://docs.sim.ai/cli/usage-data'
/**
* Printed once, the first time reporting would happen on an interactive
* terminal, the way the Vercel and Next.js notices are. The run that shows it
* is not reported, so nothing leaves the machine before the user has read
* that something will.
*/
export const FIRST_RUN_NOTICE = [
'Sim collects anonymous usage data to improve the CLI: which commands run, whether',
'they succeed, and how long they take. Nothing you type is sent.',
`Learn more: ${USAGE_DATA_DOCS_URL}`,
'Turn it off: sim telemetry disable',
'',
].join('\n')
/**
* The event's properties. Kept in one place so the documentation page can be
* checked against it and a new property is a deliberate addition here.
*/
export interface CommandEventProperties {
/** Analytics library conventions: the reporting client and its version. */
$lib: typeof LIBRARY_NAME
$lib_version: string
/** Events are anonymous and must not create a person profile per device. */
$process_person_profile: false
/** Commands close in time share a session, so a sequence of commands can be read back. */
$session_id: string
/** The command's position within its session. */
session_sequence: number
/** The same surface name the server stamps on requests carrying `X-Sim-Client-Info`. */
surface: 'cli'
/** The command's path, such as `workflows list` — never its arguments. */
command: string
/** The long names of flags that were typed, such as `--output`; never their values. */
flags: string[]
/** How many positional arguments were given; never what they were. */
arg_count: number
exit_code: number
/** Time from process start to completion, in milliseconds. */
duration_ms: number
/** The failure's class name, such as `SimApiError`; never its message. */
error_name?: string
/** The API's machine-readable error code, such as `NOT_FOUND`. */
error_code?: string
http_status?: number
cli_version: string
node_version: string
os: string
arch: string
/** Whether stdout was a terminal, which separates people from scripts. */
is_tty: boolean
is_ci: boolean
/** The AI coding agent driving this shell, or `none` when none was detected. */
coding_agent: string
/** Whether the profile targets Sim's hosted deployment or a self-hosted one; never the address. */
endpoint_kind?: 'hosted' | 'self_hosted'
}
export interface InvocationOutcome {
exitCode: number
error?: unknown
}
export interface CommandTelemetryOptions {
env?: NodeJS.ProcessEnv
ingestTarget?: () => IngestTarget | undefined
send?: typeof sendCapture
now?: () => Date
/** Milliseconds since the process started, for the duration. */
elapsed?: () => number
/** Whether stdout is a terminal: the `is_tty` property, which separates people from scripts. */
stdoutIsTty?: boolean
/** Whether stderr is a terminal: where the notice would go, so whether anyone would see it. */
stderrIsTty?: boolean
/** Where the first-run notice goes; stderr, so piped output stays clean. */
write?: (message: string) => void
/** Registers the listener that reports when the process ends; the real process by default. */
onExit?: (listener: (exitCode: number) => void) => void
}
export interface CommandTelemetry {
/** Installs the hook that records which command is about to run, and the exit listener that reports it. */
observe(program: Command): void
/**
* Reports the recorded command, given how it ended. Idempotent, and a no-op
* when nothing was recorded: the entrypoint calls it with the failure it
* explained, and the exit listener calls it for every other way out.
*/
complete(outcome: InvocationOutcome): void
}
interface RecordedInvocation {
action: Command
command: string
flags: string[]
argCount: number
state: TelemetryState
/** Set when this run printed the first-run notice and is therefore not reported. */
noticeShown: boolean
}
/**
* The flags typed on the command line, on the leaf and every ancestor, so
* root globals like `--output` count. Only source `cli`: a value that came
* from the environment or a default was not something the user typed here.
*/
function typedFlags(command: Command): string[] {
const flags = new Set<string>()
for (let current: Command | null = command; current; current = current.parent) {
for (const option of current.options) {
if (current.getOptionValueSource(option.attributeName()) !== 'cli') continue
const name = option.long ?? option.short
if (name) flags.add(name)
}
}
return [...flags]
}
/** Whether the profile points at Sim's hosted deployment; `undefined` when no profile resolves. */
function endpointKind(command: Command): CommandEventProperties['endpoint_kind'] | undefined {
try {
const hostname = new URL(profileFrom(command).endpoint).hostname
return hostname === 'sim.ai' || hostname.endsWith('.sim.ai') ? 'hosted' : 'self_hosted'
} catch {
return undefined
}
}
function failureProperties(
error: unknown
): Pick<CommandEventProperties, 'error_name' | 'error_code' | 'http_status'> {
if (error === undefined) return {}
if (!(error instanceof Error)) return { error_name: 'unknown' }
const properties: ReturnType<typeof failureProperties> = { error_name: error.name }
if (error instanceof SimApiError) {
if (error.code) properties.error_code = error.code
if (error.status > 0) properties.http_status = error.status
}
return properties
}
const listenForProcessExit = (listener: (exitCode: number) => void): void => {
process.once('exit', listener)
}
export function createCommandTelemetry(options: CommandTelemetryOptions = {}): CommandTelemetry {
const env = options.env ?? process.env
const ingestTarget = options.ingestTarget ?? builtInIngestTarget
const send = options.send ?? sendCapture
const now = options.now ?? (() => new Date())
const elapsed = options.elapsed ?? (() => performance.now())
const stdoutIsTty = options.stdoutIsTty ?? process.stdout.isTTY === true
const stderrIsTty = options.stderrIsTty ?? process.stderr.isTTY === true
const write = options.write ?? ((message: string) => void process.stderr.write(message))
const onExit = options.onExit ?? listenForProcessExit
let recorded: RecordedInvocation | undefined
/**
* Whether this run could report at all, decided from the environment and the
* build alone so a run that cannot report never touches the state file.
*/
function isReportable(state: Pick<TelemetryState, 'enabled'>): boolean {
return telemetryStatus({ env, state, configured: ingestTarget() !== undefined }).enabled
}
/**
* Shows the notice on the first interactive run that would report, and
* remembers having done so. Gated on stderr, where it is written: not in CI,
* where nobody is reading, and not when stderr is redirected, where it would
* land in a log — while `sim … | jq`, which redirects only stdout, still
* shows it.
*/
function showNoticeIfDue(state: TelemetryState): boolean {
if (state.noticeShownAt || !stderrIsTty || isCi(env)) return false
write(FIRST_RUN_NOTICE)
writeTelemetryState({ ...state, noticeShownAt: now().toISOString() })
return true
}
function complete(outcome: InvocationOutcome): void {
const invocation = recorded
recorded = undefined
if (!invocation || invocation.noticeShown) return
const target = ingestTarget()
if (!target) return
/**
* Re-read rather than reuse the snapshot from before the command ran: an
* opt-out saved meanwhile — `sim telemetry disable` in another terminal,
* during a long command — must win, and the write below must not put the
* stale snapshot back over it.
*/
const state = loadTelemetryState()
if (!isReportable(state)) return
const timestamp = now()
const session = nextSession(state, timestamp)
writeTelemetryState({ ...state, session })
const properties: CommandEventProperties = {
$lib: LIBRARY_NAME,
$lib_version: cliVersion(),
$process_person_profile: false,
$session_id: session.id,
session_sequence: session.sequence,
surface: 'cli',
command: invocation.command,
flags: invocation.flags,
arg_count: invocation.argCount,
exit_code: outcome.exitCode,
duration_ms: Math.round(elapsed()),
...failureProperties(outcome.error),
cli_version: cliVersion(),
node_version: process.versions.node,
os: process.platform,
arch: process.arch,
is_tty: stdoutIsTty,
is_ci: isCi(env),
coding_agent: detectCodingAgent(env) ?? NO_CODING_AGENT,
}
const kind = endpointKind(invocation.action)
if (kind) properties.endpoint_kind = kind
send(target, {
api_key: target.key,
event: COMMAND_EVENT,
distinct_id: state.deviceId,
timestamp: timestamp.toISOString(),
properties,
} satisfies CaptureRequest<CommandEventProperties>)
}
return {
observe(program) {
program.hook('preAction', (_root, action) => {
const path = commandPath(action)
if (path[0] === EXCLUDED_ROOT_COMMAND || !isReportable({})) return
const state = loadTelemetryState()
if (!isReportable(state)) return
recorded = {
action,
command: path.join(' '),
flags: typedFlags(action),
argCount: action.args.length,
state,
noticeShown: showNoticeIfDue(state),
}
})
/**
* Commands end in more ways than one: a handler that calls `process.exit`
* itself, a `process.exitCode` set on the way out, or the entrypoint's
* own exit. Reporting from the exit event sees all of them, and the
* sender is a synchronous spawn, which is what an exit listener allows.
*/
onExit((exitCode) => complete({ exitCode }))
},
complete,
}
}