Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions apps/docs/openapi-v2-workflows.json
Original file line number Diff line number Diff line change
Expand Up @@ -15969,7 +15969,7 @@
"failed",
"cancelled"
],
"description": "Current or terminal run status. `redacting` is transient, reported while a finished run's output is being scrubbed. `paused` means the run is waiting at a human or time pause, or a resume attempt did not complete. The single-run response provides the pause kind, resume time, and any automatic-resume waiting reason."
"description": "Current or terminal run status. `redacting` is transient, reported while a finished run's output is being scrubbed. `paused` means the run has a human or time pause that can still be resumed; once the run has finished and every pause point has resumed or failed, it reports the finished status. The single-run response provides the pause kind, resume time, and any automatic-resume waiting reason."
},
"trigger": {
"type": "string",
Expand Down Expand Up @@ -16468,7 +16468,7 @@
"cancelled",
"queued"
],
"description": "Current or terminal run status. `redacting` is transient, reported while a finished run's output is being scrubbed. `paused` means the run is waiting at a human or time pause, or a resume attempt did not complete. The single-run response provides the pause kind, resume time, and any automatic-resume waiting reason."
"description": "Current or terminal run status. `redacting` is transient, reported while a finished run's output is being scrubbed. `paused` means the run has a human or time pause that can still be resumed; once the run has finished and every pause point has resumed or failed, it reports the finished status. The single-run response provides the pause kind, resume time, and any automatic-resume waiting reason."
},
"trigger": {
"anyOf": [
Expand Down
2 changes: 1 addition & 1 deletion apps/sim/lib/api/contracts/v2/workflows.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1983,7 +1983,7 @@ export const v2ResumeWorkflowContract = defineRouteContract({
})

const RUN_STATUS_DESCRIPTION =
"Current or terminal run status. `redacting` is transient, reported while a finished run's output is being scrubbed. `paused` means the run is waiting at a human or time pause, or a resume attempt did not complete. The single-run response provides the pause kind, resume time, and any automatic-resume waiting reason."
"Current or terminal run status. `redacting` is transient, reported while a finished run's output is being scrubbed. `paused` means the run has a human or time pause that can still be resumed; once the run has finished and every pause point has resumed or failed, it reports the finished status. The single-run response provides the pause kind, resume time, and any automatic-resume waiting reason."

/**
* The list projection passes `workflow_execution_logs.status` through except where it
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@
* PostgreSQL: the pause's reviewer content is stored and read back with the run's resolved
* secret redacted, and the run detail withholds that content on the same terms as the run's
* output, while the pause's input fields stay readable. The workspace's paused-run list shows
* each point on exactly the same terms.
* each point on exactly the same terms. A run whose resumed execution failed reads failed, not
* paused, once no point is left to resume.
*/
import { db } from '@sim/db'
import {
Expand Down Expand Up @@ -32,6 +33,7 @@ import { encryptSecret } from '@/lib/core/security/encryption'
import { executeWorkflowOperation } from '@/lib/workflows/application/execute-workflow'
import { listPausedRuns } from '@/lib/workflows/application/list-paused-runs'
import { readWorkflowRun } from '@/lib/workflows/application/read-workflow-run'
import { listWorkflowExecutions } from '@/lib/workflows/executor/execution-queries'
import { PauseResumeManager } from '@/lib/workflows/executor/human-in-the-loop-manager'
import type { PausePoint } from '@/executor/types'

Expand Down Expand Up @@ -475,3 +477,55 @@ describe("the workspace's paused-run list", () => {
expect(second.runs[0].pausedAt.getTime()).toBeLessThanOrEqual(first.runs[0].pausedAt.getTime())
})
})

describe('a run whose resumed execution failed', () => {
/** What a failed resume leaves behind: the log failed, its point failed, the record still open. */
async function settleAsFailed(runId: string) {
const [row] = await db
.select({ id: pausedExecutions.id, pausePoints: pausedExecutions.pausePoints })
.from(pausedExecutions)
.where(eq(pausedExecutions.executionId, runId))
const points = row.pausePoints as Record<string, PausePoint>
await db
.update(pausedExecutions)
.set({
pausePoints: Object.fromEntries(
Object.entries(points).map(([id, point]) => [id, { ...point, resumeStatus: 'failed' }])
),
})
.where(eq(pausedExecutions.id, row.id))
await db
.update(workflowExecutionLogs)
.set({ status: 'failed', level: 'error', endedAt: new Date() })
.where(eq(workflowExecutionLogs.executionId, runId))
}

async function listedStatus(runId: string) {
const rows = await listWorkflowExecutions({ workflowId, limit: 100, order: 'desc' })
return rows.data.find((row) => row.executionId === runId)?.status
}

it('reads failed in the run detail and the run list once no point can be resumed', async () => {
const runId = await pausedRun()
await settleAsFailed(runId)

const run = await readRun(owner, runId, { includeOutput: false })
expect(run).toMatchObject({ status: 'failed', paused: null, pausePoints: null })
expect(await listedStatus(runId)).toBe('failed')
})

it('still reads paused while a point is waiting, even though the log has finished', async () => {
const runId = await pausedRun()
await db
.update(workflowExecutionLogs)
.set({ status: 'failed', level: 'error', endedAt: new Date() })
.where(eq(workflowExecutionLogs.executionId, runId))

const run = await readRun(owner, runId, { includeOutput: false })
expect(run.status).toBe('paused')
expect(run.pausePoints).toEqual([
expect.objectContaining({ contextId: 'approval', status: 'paused' }),
])
expect(await listedStatus(runId)).toBe('paused')
})
})
15 changes: 14 additions & 1 deletion apps/sim/lib/workflows/executor/execution-queries.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import { db } from '@sim/db'
import { pausedExecutions, workflowExecutionLogs } from '@sim/db/schema'
import { and, asc, desc, eq, gt, gte, lt, lte, or, sql } from 'drizzle-orm'
import { PAUSE_POINT_ROWS } from '@/lib/workflows/executor/pause-point-sql'

export type WorkflowExecutionStatus =
| 'pending'
Expand All @@ -26,8 +27,20 @@ export interface ListWorkflowExecutionsInput {
order: 'asc' | 'desc'
}

/**
* An open pause record reads `paused` unless the log has finished and no point is left to resume,
* the same rule `readWorkflowExecutionStatus` applies to a single run.
*/
const executionStatus = sql<string>`CASE
WHEN ${pausedExecutions.status} IN ('paused', 'partially_resumed') THEN 'paused'
WHEN ${pausedExecutions.status} IN ('paused', 'partially_resumed')
AND NOT (
${workflowExecutionLogs.status} IN ('completed', 'failed', 'cancelled')
AND NOT EXISTS (
select 1 from ${PAUSE_POINT_ROWS}
where coalesce(point.value ->> 'resumeStatus', 'paused') NOT IN ('resumed', 'failed')
)
)
THEN 'paused'
ELSE ${workflowExecutionLogs.status}
END`

Expand Down
115 changes: 115 additions & 0 deletions apps/sim/lib/workflows/executor/execution-status.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ vi.mock('@/lib/workflows/executor/paused-execution-metadata', () => ({
import { FunctionalOutputsUnavailableError } from '@/lib/logs/execution/functional-outputs'
import { isWaitingHumanPausePoint } from '@/lib/workflows/executor/execution-state'
import {
getProjectedWorkflowExecutionStatus,
getWorkflowExecutionStatus,
readPausedRunPoints,
} from '@/lib/workflows/executor/execution-status'
Expand Down Expand Up @@ -665,6 +666,120 @@ describe('getWorkflowExecutionStatus polls that read no execution data', () => {
})
})

describe('getWorkflowExecutionStatus once a pause has settled', () => {
const FAILED_TIME_POINT = {
contextId: 'wait-five',
blockId: 'wait-five',
registeredAt: '2026-08-05T12:00:01.000Z',
resumeStatus: 'failed',
pauseKind: 'time',
resumeAt: '2026-08-05T12:05:01.000Z',
}

/** A run whose resumed execution failed: the log is terminal, the pause record still open. */
function queueFailedRunWithOpenPause(pausePoints: Record<string, unknown>) {
queueTableRows(schemaMock.workflowExecutionLogs, [
{
executionId: 'execution-1',
workflowId: 'workflow-1',
workspaceId: 'workspace-1',
status: 'failed',
level: 'error',
trigger: 'api',
startedAt: new Date('2026-08-05T12:00:00.000Z'),
endedAt: new Date('2026-08-05T12:05:03.000Z'),
totalDurationMs: 303000,
executionData: {},
costTotal: null,
},
])
queueTableRows(schemaMock.resumeQueue, [])
queueTableRows(schemaMock.pausedExecutions, [
{
id: 'paused-execution-1',
status: 'paused',
pausePoints,
metadata: {},
resumedCount: 0,
pausedAt: new Date('2026-08-05T12:00:01.000Z'),
nextResumeAt: null,
},
])
}

beforeEach(() => {
resetDbChainMock()
mockGetJob.mockResolvedValue(null)
mockMaterializeForDisplayWithBlockOutputs.mockResolvedValue({
executionData: {
error: 'Lookup: upstream returned 500',
traceSpans: [
{
id: 'span-1',
name: 'Lookup',
type: 'api',
status: 'error',
blockId: 'lookup',
executionOrder: 2,
},
],
},
blockOutputs: new Map(),
})
})

it('reports a run whose resumed execution failed as failed, with its error and failing block', async () => {
queueFailedRunWithOpenPause({ 'wait-five': FAILED_TIME_POINT })

const projected = await getProjectedWorkflowExecutionStatus(input)

expect(projected?.status).toMatchObject({
status: 'failed',
paused: null,
error: 'Lookup: upstream returned 500',
})
expect(projected?.pausePoints).toBeNull()
expect(projected?.failedBlock).toMatchObject({ blockId: 'lookup', blockName: 'Lookup' })
})

it.each([
[
'a human pause',
{
contextId: 'approval',
blockId: 'approval',
resumeStatus: 'paused',
pauseKind: 'human',
resumeAt: null,
},
],
[
'a time pause',
{
contextId: 'wait-ten',
blockId: 'wait-ten',
resumeStatus: 'paused',
pauseKind: 'time',
resumeAt: '2026-08-05T12:10:01.000Z',
},
],
])('still reads paused while %s is waiting beside the failed one', async (_, waiting) => {
queueFailedRunWithOpenPause({ 'wait-five': FAILED_TIME_POINT, [waiting.contextId]: waiting })

const status = await getWorkflowExecutionStatus(input)

expect(status).toMatchObject({
status: 'paused',
error: null,
paused: {
contextId: waiting.contextId,
pauseKind: waiting.pauseKind,
resumeAt: waiting.resumeAt,
},
})
})
})

describe('readPausedRunPoints cap', () => {
beforeEach(() => {
resetDbChainMock()
Expand Down
14 changes: 13 additions & 1 deletion apps/sim/lib/workflows/executor/execution-status.ts
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,11 @@ function pickEarliestPausePoint(points: PausePoint[]): PausePoint | null {
}, null)
}

/** A point no resume can act on any more: it resumed, or its resumed execution failed. */
function isSettledPausePoint(point: PausePoint): boolean {
return point.resumeStatus === 'resumed' || point.resumeStatus === 'failed'
}

function extractError(executionData: unknown): string | null {
if (!executionData || typeof executionData !== 'object') return null
const data = executionData as ExecutionDataShape
Expand Down Expand Up @@ -519,8 +524,15 @@ async function readWorkflowExecutionStatus(
)
.limit(1)

/**
* An open pause record makes the run read `paused` unless the log has finished and no point
* is left to resume: a resume that ran and failed marks its point `failed` and fails the log
* but leaves the record open, and that point can never be resumed again.
*/
const isCurrentlyPaused =
!!pausedRow && (pausedRow.status === 'paused' || pausedRow.status === 'partially_resumed')
!!pausedRow &&
(pausedRow.status === 'paused' || pausedRow.status === 'partially_resumed') &&
!(hasTerminalLog && normalizePausePoints(pausedRow.pausePoints).every(isSettledPausePoint))

let status: WorkflowExecutionStatusResponse['status']
if (isCurrentlyPaused) {
Expand Down
21 changes: 21 additions & 0 deletions apps/sim/lib/workflows/executor/pause-point-sql.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
import { pausedExecutions } from '@sim/db/schema'
import { sql } from 'drizzle-orm'

/**
* Every pause point of a `paused_executions` row, as the derived table `point(value)`.
*
* `pause_points` is an object keyed by context id, or an array on legacy rows; both shapes are
* read, as `normalizePausePoints` reads them, and each set function only ever receives the shape
* it accepts.
*/
export const PAUSE_POINT_ROWS = sql`(
select value from jsonb_each(
case when jsonb_typeof(${pausedExecutions.pausePoints}) = 'object'
then ${pausedExecutions.pausePoints} else '{}'::jsonb end
)
union all
select value from jsonb_array_elements(
case when jsonb_typeof(${pausedExecutions.pausePoints}) = 'array'
then ${pausedExecutions.pausePoints} else '[]'::jsonb end
)
) as point(value)`
22 changes: 3 additions & 19 deletions apps/sim/lib/workflows/executor/paused-runs.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import {
resumeKeyset,
textKey,
} from '@/lib/api/list-query'
import { PAUSE_POINT_ROWS } from '@/lib/workflows/executor/pause-point-sql'

/** A paused run of an active workflow, with the raw `pause_points` its detail is built from. */
interface WorkspacePausedRunRow {
Expand All @@ -35,25 +36,8 @@ const PAUSED_RUN_KEYS: readonly KeysetKey<WorkspacePausedRunRow>[] = [
const POINT_WAITS_ON_HUMAN = sql`coalesce(point.value ->> 'pauseKind', 'human') <> 'time'
and coalesce(point.value ->> 'resumeStatus', 'paused') = 'paused'`

/**
* A paused run with a pause point still waiting on a human. `pause_points` is an object keyed
* by context id, or an array on legacy rows; both shapes are read, as `normalizePausePoints`
* reads them, and each set function only ever receives the shape it accepts.
*/
const WAITS_ON_HUMAN = sql`exists (
select 1 from (
select value from jsonb_each(
case when jsonb_typeof(${pausedExecutions.pausePoints}) = 'object'
then ${pausedExecutions.pausePoints} else '{}'::jsonb end
)
union all
select value from jsonb_array_elements(
case when jsonb_typeof(${pausedExecutions.pausePoints}) = 'array'
then ${pausedExecutions.pausePoints} else '[]'::jsonb end
)
) as point(value)
where ${POINT_WAITS_ON_HUMAN}
)`
/** A paused run with a pause point still waiting on a human. */
const WAITS_ON_HUMAN = sql`exists (select 1 from ${PAUSE_POINT_ROWS} where ${POINT_WAITS_ON_HUMAN})`

/** The cursor that resumes the list after `row`. */
export function pausedRunCursorKeys(row: WorkspacePausedRunRow): CursorKey[] {
Expand Down
3 changes: 2 additions & 1 deletion packages/sim-cli/src/commands/credentials.ts
Original file line number Diff line number Diff line change
Expand Up @@ -186,7 +186,8 @@ function withKeylessRetryHint(error: unknown, providerId: string): unknown {
error.status,
error.code,
error.details,
error.exitCode
error.exitCode,
error.requestId
)
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -152,7 +152,8 @@ export async function waitWorkspaceOperation(
requestId: report?.requestId,
applied: report?.applied,
},
failure.exitCode
failure.exitCode,
failure.requestId
)
}
if (report.status !== 'processing') return { report, timedOut: false }
Expand Down
1 change: 1 addition & 0 deletions packages/sim-cli/src/embed.ts
Original file line number Diff line number Diff line change
Expand Up @@ -196,6 +196,7 @@ function renderEmbeddedError(ctx: EmbedContext, error: unknown): number {
if (error.details !== undefined) {
for (const line of formatApiErrorDetails(error.details)) ctx.stderr.diagnostic(sanitize(line))
}
if (error.requestId) ctx.stderr.diagnostic(` request id: ${sanitize(error.requestId)}`)
return error.exitCode
}
ctx.stderr.diagnostic(`Error: ${sanitize(getErrorMessage(error))}`)
Expand Down
Loading
Loading