Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -259,7 +259,7 @@ Run failures are reported in the result, not as HTTP errors: branch on `status`

## 10. Debug a failed run [#debugging]

A synchronous execute response names the failing block in `error.blockId`, `error.blockName`, and `error.blockType`. The polled run resource cannot attribute a block; read the run's trace spans instead:
Both the execute response and the polled run resource name the failing block in `error.blockId`, `error.blockName`, and `error.blockType`. The run resource reads these from the run's trace, so they are absent when the trace is unavailable or hidden by your organization's permission groups, when no failed block in the trace recorded the run error, and for a timeout, cancellation, usage-limit, or invalid-input failure. Read the trace spans for the full picture:

<Tabs items={['curl', 'CLI']}>
<Tab value="curl">
Expand Down
2 changes: 1 addition & 1 deletion apps/docs/openapi-v2-workflows.json
Original file line number Diff line number Diff line change
Expand Up @@ -15539,7 +15539,7 @@
"description": "Stable machine-readable execution failure code. `BLOCK_EXECUTION_FAILED` and `CHILD_WORKFLOW_FAILED` are reported only where block attribution is available; elsewhere a block-level failure is reported as `EXECUTION_FAILED`."
},
"blockId": {
"description": "Identifier of the failing block, when attributable. Get workflow run reads it from the run trace, so it is absent there when the trace is unavailable or hidden by organization permission groups, and for a timeout, cancellation, usage-limit, or invalid-input failure.",
"description": "Identifier of the failing block, when attributable. Get workflow run reads it from the run trace, so it is absent there when the trace is unavailable or hidden by organization permission groups, when no failed block in the trace recorded the run error, and for a timeout, cancellation, usage-limit, or invalid-input failure.",
"type": "string"
},
"blockName": {
Expand Down
2 changes: 1 addition & 1 deletion apps/sim/app/cli/auth/cli-auth-view.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ export function CliAuthView() {
*/
const keyLifetime =
request.expiresInDays === null
? ''
? ' that never expires'
: ` that expires in ${request.expiresInDays} ${request.expiresInDays === 1 ? 'day' : 'days'}`
const keyDescription = `${request.access === 'read' ? 'a read-only' : 'a'} personal key tied to your account${keyLifetime}`

Expand Down
61 changes: 61 additions & 0 deletions apps/sim/executor/utils/errors.test.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
import { describe, expect, it } from 'vitest'
import { v2ExecutionErrorSchema } from '@/lib/api/contracts/v2/workflows'
import { HttpError } from '@/lib/core/utils/http-error'
import { findFailedTraceBlock } from '@/lib/logs/execution/trace-spans/trace-spans'
import type { TraceSpan } from '@/lib/logs/types'
import type { ExecutionResult } from '@/executor/types'
import {
buildBlockExecutionError,
Expand Down Expand Up @@ -274,6 +276,65 @@ describe('classifyRecordedExecutionError', () => {
})
})

/** A recorded trace holding one failed block span, as a polled run reads it back. */
const traceWithFailure = (failed: Partial<TraceSpan>): TraceSpan[] => [
{
id: 'workflow-execution',
name: 'Workflow Execution',
type: 'workflow',
duration: 1,
startTime: '2024-01-01T10:00:00.000Z',
endTime: '2024-01-01T10:00:01.000Z',
status: 'error',
children: [
{
id: 'failed-span',
name: 'Failed',
type: 'function',
duration: 1,
startTime: '2024-01-01T10:00:00.000Z',
endTime: '2024-01-01T10:00:01.000Z',
status: 'error',
...failed,
},
],
},
]
const classifyFromTrace = (runError: string, failed: Partial<TraceSpan>) =>
classifyRecordedExecutionError(
runError,
findFailedTraceBlock(traceWithFailure(failed), runError)
)

it('attributes a loop failure, recorded without a name prefix, to the loop', () => {
const runError = 'ForEach loop resolution failed: items is not an array'
expect(
classifyFromTrace(runError, {
blockId: 'loop-1',
name: 'Each lead',
type: 'loop',
output: { error: runError },
})
).toEqual({
message: runError,
code: 'BLOCK_EXECUTION_FAILED',
blockId: 'loop-1',
blockName: 'Each lead',
blockType: 'loop',
})
})

it('does not attribute a run error to a failed block that did not record it', () => {
expect(
classifyFromTrace('Service unavailable', {
blockId: 'block-1',
name: 'Lookup',
type: 'api',
output: { error: 'upstream returned 500' },
})
).toEqual({ message: 'Service unavailable', code: 'EXECUTION_FAILED' })
})

it('reports EXECUTION_FAILED when the trace names no block', () => {
expect(classifyRecordedExecutionError('Lookup: upstream returned 500', null)).toEqual({
message: 'Lookup: upstream returned 500',
Expand Down
11 changes: 6 additions & 5 deletions apps/sim/executor/utils/errors.ts
Original file line number Diff line number Diff line change
Expand Up @@ -376,11 +376,12 @@ export function classifyExecutionError(

/**
* Classifies a run's persisted error message, attributing it to the failing
* block its recorded trace names. A message headed by that block's name is the
* block's own failure and is classified with the block attached, whatever the
* name says. A message that does not name it stays unattributed when its text
* classifies as run-level (a timeout, cancellation, usage limit, or invalid
* input), and without a block this reports what the message alone supports.
* block its recorded trace names as the owner of that message (see
* `findFailedTraceBlock`). A message headed by that block's name is the block's
* own failure and is classified with the block attached, whatever the name says.
* A message that does not name it stays unattributed when its text classifies as
* run-level (a timeout, cancellation, usage limit, or invalid input), and without
* a block this reports what the message alone supports.
*/
export function classifyRecordedExecutionError(
message: string,
Expand Down
7 changes: 4 additions & 3 deletions apps/sim/lib/api/contracts/v2/workflows.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1550,8 +1550,9 @@ export const v2ExecutionErrorSchema = z
* The synchronous execute and resume responses attribute the block at
* execution time. The polled run resource keeps only the error *string*, so
* it reads the block back from the run's recorded trace: the unhandled
* failure the stored error names, else the one that ended first, and none
* when concurrent failures cannot be told apart. When that trace is unavailable (not
* failure that recorded the stored error (named by it, or with the same
* text), the one that ended first among several, and none when they cannot
* be told apart or no failed block recorded it. When that trace is unavailable (not
* recorded, compacted, or withheld by the viewer's permission group) the
* three fields are absent and a block-level failure reports
* `EXECUTION_FAILED`. A failure its message classifies as run-level
Expand All @@ -1562,7 +1563,7 @@ export const v2ExecutionErrorSchema = z
.string()
.optional()
.describe(
'Identifier of the failing block, when attributable. Get workflow run reads it from the run trace, so it is absent there when the trace is unavailable or hidden by organization permission groups, and for a timeout, cancellation, usage-limit, or invalid-input failure.'
'Identifier of the failing block, when attributable. Get workflow run reads it from the run trace, so it is absent there when the trace is unavailable or hidden by organization permission groups, when no failed block in the trace recorded the run error, and for a timeout, cancellation, usage-limit, or invalid-input failure.'
),
blockName: z
.string()
Expand Down
9 changes: 3 additions & 6 deletions apps/sim/lib/logs/execution/trace-spans/trace-spans.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1546,14 +1546,11 @@ describe('findFailedTraceBlock', () => {
expect(findFailedTraceBlock(concurrentFailures(), 'Agent B: timeout')?.blockId).toBe('b')
})

it('falls back to the failure that ended first when the error names no block', () => {
expect(findFailedTraceBlock(concurrentFailures(), 'upstream returned 500')?.blockId).toBe('a')
})

it('names no block when concurrent failures cannot be told apart', () => {
const output = { error: 'upstream returned 500' }
const spans = root([
span({ blockId: 'a', name: 'Agent A', status: 'error' }),
span({ blockId: 'b', name: 'Agent B', status: 'error' }),
span({ blockId: 'a', name: 'Agent A', status: 'error', output }),
span({ blockId: 'b', name: 'Agent B', status: 'error', output }),
])

expect(findFailedTraceBlock(spans, 'upstream returned 500')).toBeNull()
Expand Down
34 changes: 22 additions & 12 deletions apps/sim/lib/logs/execution/trace-spans/trace-spans.ts
Original file line number Diff line number Diff line change
Expand Up @@ -231,6 +231,8 @@ export interface FailedTraceBlock {
interface FailedSpanCandidate {
block: FailedTraceBlock
endedAt: number
/** The block log's error, which the span builder merges into its output. */
recordedError: string | undefined
}

/** The innermost unhandled-error block spans: a container whose failure came from a block inside it yields that block. */
Expand All @@ -252,6 +254,10 @@ function collectFailedBlockSpans(nodes: unknown, into: FailedSpanCandidate[]): v
blockType,
},
endedAt: typeof span.endTime === 'string' ? Date.parse(span.endTime) : Number.NaN,
recordedError:
isRecordLike(span.output) && typeof span.output.error === 'string'
? span.output.error
: undefined,
})
}
}
Expand All @@ -260,11 +266,14 @@ function collectFailedBlockSpans(nodes: unknown, into: FailedSpanCandidate[]): v
/**
* The block a recorded run failed in, read back from its persisted trace spans.
*
* A run stores the error of the failure that settled first, while branches already
* in flight may fail after it. So the span whose block name heads the stored error
* (`Block name: message`, as a block failure is recorded) is the one named; failing
* that, the failure that ended first; and when neither singles one out, none, so a
* message is never paired with another block's identity.
* Given the run's stored error, only a block that owns it is named: one whose name
* heads it (`Block name: message`, as a block failure is recorded) or whose own
* recorded error is exactly it, as a loop or parallel failure is recorded without a
* prefix. A run stores the error of the failure that settled first, while branches
* already in flight may fail after it, and a run can fail outside any block while
* one is failing; so a failed block that does not own the error is never named, and
* among several owners the one that ended first is, or none when they cannot be told
* apart. Without a run error, the failure that ended first is named.
*
* Spans are walked as {@link hasUnhandledError} walks them, including under a block
* that succeeded, and the same boundaries hold: a successful boundary block's
Expand All @@ -273,17 +282,18 @@ function collectFailedBlockSpans(nodes: unknown, into: FailedSpanCandidate[]): v
* block that ran it, as it is at execution time, and a custom block's source workflow
* is never named to its consumer.
*
* Reads stored, untyped spans structurally and never their input or output. Null when
* the trace has no such span, including when it was not recorded, was compacted, or is
* unavailable.
* Reads stored, untyped spans structurally, and of their output only the recorded
* error. Null when the trace has no such span, including when it was not recorded,
* was compacted, or is unavailable.
*/
export function findFailedTraceBlock(spans: unknown, runError?: string): FailedTraceBlock | null {
const candidates: FailedSpanCandidate[] = []
collectFailedBlockSpans(spans, candidates)
const named = runError
? candidates.filter((span) => errorNamesBlock(runError, span.block.blockName))
: []
const pool = named.length > 0 ? named : candidates
const pool = runError
? candidates.filter(
(span) => errorNamesBlock(runError, span.block.blockName) || span.recordedError === runError
)
: candidates
if (pool.length <= 1) return pool[0]?.block ?? null
if (!pool.every((span) => Number.isFinite(span.endedAt))) return null
const [first, second] = pool.sort((a, b) => a.endedAt - b.endedAt)
Expand Down
Loading