Skip to content

fix(executor): resolve saved model credentials inside each fallback attempt in Router and Evaluator #9968

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
waleedlatif1 merged 1 commit into from
Nov 13, 2026
+263 −101
Merged

fix(executor): resolve saved model credentials inside each fallback attempt in Router and Evaluator #9968

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions apps/docs/openapi-v2-knowledge.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6933,7 +6933,7 @@
"type": "null"
}
],
"description": "OAuth credential identifier, or the saved credential an `apiKey` source syncs with (see `auth.savedCredential`); null for API-key and unauthenticated sources."
"description": "OAuth credential identifier, or the saved credential an `apiKey` source syncs with (see `auth.savedCredential`); null for unauthenticated sources and for `apiKey` sources that use an inline API key."
},
"sourceConfig": {
"type": "object",
Expand Down Expand Up @@ -7331,7 +7331,7 @@
"type": "null"
}
],
"description": "OAuth credential identifier, or the saved credential an `apiKey` source syncs with (see `auth.savedCredential`); null for API-key and unauthenticated sources."
"description": "OAuth credential identifier, or the saved credential an `apiKey` source syncs with (see `auth.savedCredential`); null for unauthenticated sources and for `apiKey` sources that use an inline API key."
},
"sourceConfig": {
"type": "object",
Expand Down
33 changes: 13 additions & 20 deletions apps/sim/executor/handlers/agent/handler.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,9 +91,12 @@ import type {
StreamingExecution,
UserFile,
} from '@/executor/types'
import { resolveBlockBedrockSession } from '@/executor/utils/bedrock-credential'
import { collectBlockData } from '@/executor/utils/block-data'
import { stringifyJSON } from '@/executor/utils/json'
import {
BLOCK_MODEL_CREDENTIAL_FIELDS,
resolveBlockModelCredentials,
} from '@/executor/utils/model-credentials'
import {
assertFallbackFamilyCredentials,
getModelFallbacks,
Expand All @@ -109,7 +112,6 @@ import type {
ResolvedSecretTraceRegistry,
} from '@/executor/utils/resolved-secret-trace-registry'
import { annotateDuplicateToolBindings } from '@/executor/utils/tool-binding-labels'
import { resolveVertexCredential } from '@/executor/utils/vertex-credential'
import { executeProviderRequest } from '@/providers'
import {
formatAttachmentSizes,
Expand Down Expand Up @@ -2710,9 +2712,7 @@ export class AgentBlockHandler implements BlockHandler {
? config.modelInputs
: omit(config.modelInputs, [
...PROVIDER_FAMILY_CREDENTIAL_FIELDS,
'vertexCredential',
'bedrockCredential',
'bedrockAuthMode',
...BLOCK_MODEL_CREDENTIAL_FIELDS,
])),
apiKey: resolveFallbackApiKey({
candidate,
Expand Down Expand Up @@ -2989,19 +2989,12 @@ export class AgentBlockHandler implements BlockHandler {
const model = providerRequest.model

try {
let finalApiKey: string | undefined = providerRequest.apiKey

if (providerId === 'vertex' && providerRequest.vertexCredential) {
finalApiKey = await resolveVertexCredential({
credentialId: providerRequest.vertexCredential,
actingUserId: ctx.userId,
workspaceId: ctx.workspaceId,
workflowId: ctx.workflowId,
executionId: ctx.executionId,
callerLabel: 'vertex-agent',
})
}
const bedrockSession = await resolveBlockBedrockSession(ctx, providerId, providerRequest)
const credentials = await resolveBlockModelCredentials(
ctx,
providerId,
providerRequest,
'vertex-agent'
)

const { blockData, blockNameMapping } = collectBlockData(ctx)
const agentMemoryRetrieval = agentConversation?.memoryId
Expand All @@ -3024,15 +3017,15 @@ export class AgentBlockHandler implements BlockHandler {
: providerRequest.tools,
temperature: providerRequest.temperature,
maxTokens: providerRequest.maxTokens,
apiKey: finalApiKey,
apiKey: credentials.apiKey ?? providerRequest.apiKey,
azureEndpoint: providerRequest.azureEndpoint,
azureApiVersion: providerRequest.azureApiVersion,
vertexProject: providerRequest.vertexProject,
vertexLocation: providerRequest.vertexLocation,
bedrockAccessKeyId: providerRequest.bedrockAccessKeyId,
bedrockSecretKey: providerRequest.bedrockSecretKey,
bedrockRegion: providerRequest.bedrockRegion,
bedrockSession,
bedrockSession: credentials.bedrockSession,
responseFormat: providerRequest.responseFormat,
workflowId: providerRequest.workflowId,
workspaceId: ctx.workspaceId,
Expand Down
45 changes: 45 additions & 0 deletions apps/sim/executor/handlers/evaluator/handler.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import {
import {
llmBedrockCredentialMock,
llmBedrockCredentialMockFns,
MockBedrockCredentialError,
} from '@sim/testing/mocks/llm-bedrock-credential.mock'
import { permissionsMock, permissionsMockFns } from '@sim/testing/mocks/permissions.mock'
import '@sim/testing/mocks/executor'
Expand Down Expand Up @@ -252,6 +253,50 @@ describe('EvaluatorBlockHandler', () => {
expect(JSON.stringify(fallbackRequest)).not.toContain('saved-secret')
})

it('scores with a fallback model when the saved AWS credential is unavailable on the final try', async () => {
mockGetProviderFromModel.mockImplementation((model: string) =>
model.startsWith('claude') ? 'anthropic' : 'bedrock'
)
mockResolveBedrockCredentialSession.mockRejectedValueOnce(
new MockBedrockCredentialError('AWS STS is unavailable', { reason: 'unavailable' })
)
mockExecuteProviderRequest.mockResolvedValueOnce({
content: '{"score1":7}',
model: 'claude-sonnet-5',
tokens: { input: 12, output: 3, total: 15 },
cost: 0,
})
mockContext.resolvedSecretTraceRegistry = new ResolvedSecretTraceRegistry()
mockContext.blockLogs = [
{
blockId: mockBlock.id,
startedAt: '',
endedAt: '',
durationMs: 0,
success: false,
executionOrder: 1,
},
]

const output = await handler.execute(
mockContext,
mockBlock,
{
...admissionInputs,
model: 'bedrock/claude',
bedrockAuthMode: 'credential',
bedrockCredential: 'aws-cred',
fallbackModels: [{ model: 'claude-sonnet-5' }],
},
{ nodeId: mockBlock.id, retry: { attempt: 2, maxTries: 2, isFinalTry: true } }
)

expect(output).toMatchObject({ model: 'claude-sonnet-5', score1: 7 })
expect(mockExecuteProviderRequest).toHaveBeenCalledTimes(1)
expect(mockExecuteProviderRequest.mock.calls[0][0]).toBe('anthropic')
expect(mockContext.blockLogs[0].modelFallbacks).toEqual(['bedrock/claude'])
})

it('fails a Bedrock evaluation whose AWS credential cannot be used instead of sending it', async () => {
mockGetProviderFromModel.mockReturnValue('bedrock')
mockResolveBedrockCredentialSession.mockRejectedValueOnce(
Expand Down
19 changes: 2 additions & 17 deletions apps/sim/executor/handlers/evaluator/handler.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ import type { BlockOutput } from '@/blocks/types'
import { validateModelProvider } from '@/ee/access-control/utils/permission-check'
import { BlockType, DEFAULTS, EVALUATOR } from '@/executor/constants'
import type { BlockHandler, BlockNodeMetadata, ExecutionContext } from '@/executor/types'
import { resolveBlockBedrockSession } from '@/executor/utils/bedrock-credential'
import { isJSONString, parseJSON, stringifyJSON } from '@/executor/utils/json'
import { executeModelRequestWithFallbacks } from '@/executor/utils/model-fallback-request'
import { projectResolvedSecretDiagnosticError } from '@/executor/utils/resolved-secret-content-projection'
Expand All @@ -19,7 +18,6 @@ import type {
ResolvedSecretInputPath,
ResolvedSecretTraceRegistry,
} from '@/executor/utils/resolved-secret-trace-registry'
import { resolveVertexCredential } from '@/executor/utils/vertex-credential'
import { resolveProxiedModelCost } from '@/providers/cost-policy'
import { isAutoModel, SIM_AUTO_MODEL_ID } from '@/providers/models'
import type { ProviderRequest } from '@/providers/types'
Expand Down Expand Up @@ -181,19 +179,6 @@ export class EvaluatorBlockHandler implements BlockHandler {
await validateModelProvider(ctx.userId, ctx.workspaceId, model, ctx)
const providerId = getProviderFromModel(model)

let finalApiKey: string | undefined = evaluatorConfig.apiKey
if (providerId === 'vertex' && evaluatorConfig.vertexCredential) {
finalApiKey = await resolveVertexCredential({
credentialId: evaluatorConfig.vertexCredential,
actingUserId: ctx.userId,
workspaceId: ctx.workspaceId,
workflowId: ctx.workflowId,
executionId: ctx.executionId,
callerLabel: 'vertex-evaluator',
})
}
const bedrockSession = await resolveBlockBedrockSession(ctx, providerId, evaluatorConfig)

try {
const providerRequest: ProviderRequest = {
model,
Expand All @@ -208,15 +193,14 @@ export class EvaluatorBlockHandler implements BlockHandler {
]),

temperature: EVALUATOR.DEFAULT_TEMPERATURE,
apiKey: finalApiKey,
apiKey: evaluatorConfig.apiKey,
azureEndpoint: inputs.azureEndpoint,
azureApiVersion: inputs.azureApiVersion,
vertexProject: evaluatorConfig.vertexProject,
vertexLocation: evaluatorConfig.vertexLocation,
bedrockAccessKeyId: evaluatorConfig.bedrockAccessKeyId,
bedrockSecretKey: evaluatorConfig.bedrockSecretKey,
bedrockRegion: evaluatorConfig.bedrockRegion,
bedrockSession,
workflowId: ctx.workflowId,
workspaceId: ctx.workspaceId,
}
Expand All @@ -231,6 +215,7 @@ export class EvaluatorBlockHandler implements BlockHandler {
providerId,
request: providerRequest,
resolvedSecretTraceRegistry: modelInputProjection.registry,
credentials: { inputs: evaluatorConfig, callerLabel: 'vertex-evaluator' },
})

const parsedContent = this.extractJSONFromResponse(
Expand Down
48 changes: 48 additions & 0 deletions apps/sim/executor/handlers/router/handler.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import {
import {
llmBedrockCredentialMock,
llmBedrockCredentialMockFns,
MockBedrockCredentialError,
} from '@sim/testing/mocks/llm-bedrock-credential.mock'
import { permissionsMock, permissionsMockFns } from '@sim/testing/mocks/permissions.mock'
import '@sim/testing/mocks/executor'
Expand Down Expand Up @@ -223,6 +224,53 @@ describe('RouterBlockHandler', () => {
expect(mockExecuteProviderRequest).not.toHaveBeenCalled()
})

it('routes with a fallback model when the saved AWS credential is unavailable on the final try', async () => {
mockGetProviderFromModel.mockImplementation((model: string) =>
model.startsWith('claude') ? 'anthropic' : 'bedrock'
)
mockResolveBedrockCredentialSession.mockRejectedValueOnce(
new MockBedrockCredentialError('AWS STS is unavailable', { reason: 'unavailable' })
)
mockExecuteProviderRequest.mockResolvedValueOnce({
content: 'target-block-1',
model: 'claude-sonnet-5',
tokens: { input: 10, output: 2, total: 12 },
cost: 0,
})
mockContext.resolvedSecretTraceRegistry = new ResolvedSecretTraceRegistry()
mockContext.blockLogs = [
{
blockId: mockBlock.id,
startedAt: '',
endedAt: '',
durationMs: 0,
success: false,
executionOrder: 1,
},
]

const output = await handler.execute(
mockContext,
mockBlock,
{
prompt: 'Pick a destination',
model: 'bedrock/claude',
bedrockAuthMode: 'credential',
bedrockCredential: 'aws-cred',
fallbackModels: [{ model: 'claude-sonnet-5' }],
},
{ nodeId: mockBlock.id, retry: { attempt: 2, maxTries: 2, isFinalTry: true } }
)

expect(output).toMatchObject({
model: 'claude-sonnet-5',
selectedPath: { blockId: 'target-block-1' },
})
expect(mockExecuteProviderRequest).toHaveBeenCalledTimes(1)
expect(mockExecuteProviderRequest.mock.calls[0][0]).toBe('anthropic')
expect(mockContext.blockLogs[0].modelFallbacks).toEqual(['bedrock/claude'])
})

it('selects the same legacy destination when a fallback provider answers', async () => {
mockExecuteProviderRequest
.mockRejectedValueOnce(new Error('overloaded'))
Expand Down
36 changes: 4 additions & 32 deletions apps/sim/executor/handlers/router/handler.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,9 @@ import {
ROUTER,
} from '@/executor/constants'
import type { BlockHandler, BlockNodeMetadata, ExecutionContext } from '@/executor/types'
import { resolveBlockBedrockSession } from '@/executor/utils/bedrock-credential'
import { executeModelRequestWithFallbacks } from '@/executor/utils/model-fallback-request'
import { refuseResolvedSecretProjection } from '@/executor/utils/resolved-secret-projection-refusal'
import type { ResolvedSecretInputPath } from '@/executor/utils/resolved-secret-trace-registry'
import { resolveVertexCredential } from '@/executor/utils/vertex-credential'
import { resolveProxiedModelCost } from '@/providers/cost-policy'
import { isAutoModel, SIM_AUTO_MODEL_ID } from '@/providers/models'
import type { ProviderRequest } from '@/providers/types'
Expand Down Expand Up @@ -116,33 +114,19 @@ export class RouterBlockHandler implements BlockHandler {
await validateModelProvider(ctx.userId, ctx.workspaceId, resolved.model, ctx)
const providerId = getProviderFromModel(resolved.model)

let finalApiKey: string | undefined = routerConfig.apiKey
if (providerId === 'vertex' && routerConfig.vertexCredential) {
finalApiKey = await resolveVertexCredential({
credentialId: routerConfig.vertexCredential,
actingUserId: ctx.userId,
workspaceId: ctx.workspaceId,
workflowId: ctx.workflowId,
executionId: ctx.executionId,
callerLabel: 'vertex-router',
})
}
const bedrockSession = await resolveBlockBedrockSession(ctx, providerId, routerConfig)

const providerRequest: ProviderRequest = {
model: resolved.model,
systemPrompt: resolved.systemPrompt,
context: JSON.stringify(messages),
temperature: ROUTER.INFERENCE_TEMPERATURE,
apiKey: finalApiKey,
apiKey: routerConfig.apiKey,
azureEndpoint: inputs.azureEndpoint,
azureApiVersion: inputs.azureApiVersion,
vertexProject: routerConfig.vertexProject,
vertexLocation: routerConfig.vertexLocation,
bedrockAccessKeyId: routerConfig.bedrockAccessKeyId,
bedrockSecretKey: routerConfig.bedrockSecretKey,
bedrockRegion: routerConfig.bedrockRegion,
bedrockSession,
workflowId: ctx.workflowId,
workspaceId: ctx.workspaceId,
}
Expand All @@ -157,6 +141,7 @@ export class RouterBlockHandler implements BlockHandler {
providerId,
request: providerRequest,
resolvedSecretTraceRegistry: modelInputProjection.registry,
credentials: { inputs: routerConfig, callerLabel: 'vertex-router' },
})

const chosenBlockId = result.content.trim().toLowerCase()
Expand Down Expand Up @@ -290,33 +275,19 @@ export class RouterBlockHandler implements BlockHandler {
await validateModelProvider(ctx.userId, ctx.workspaceId, resolved.model, ctx)
const providerId = getProviderFromModel(resolved.model)

let finalApiKey: string | undefined = routerConfig.apiKey
if (providerId === 'vertex' && routerConfig.vertexCredential) {
finalApiKey = await resolveVertexCredential({
credentialId: routerConfig.vertexCredential,
actingUserId: ctx.userId,
workspaceId: ctx.workspaceId,
workflowId: ctx.workflowId,
executionId: ctx.executionId,
callerLabel: 'vertex-router',
})
}
const bedrockSession = await resolveBlockBedrockSession(ctx, providerId, routerConfig)

const providerRequest: ProviderRequest = {
model: resolved.model,
systemPrompt: resolved.systemPrompt,
context: JSON.stringify(messages),
temperature: ROUTER.INFERENCE_TEMPERATURE,
apiKey: finalApiKey,
apiKey: routerConfig.apiKey,
azureEndpoint: inputs.azureEndpoint,
azureApiVersion: inputs.azureApiVersion,
vertexProject: routerConfig.vertexProject,
vertexLocation: routerConfig.vertexLocation,
bedrockAccessKeyId: routerConfig.bedrockAccessKeyId,
bedrockSecretKey: routerConfig.bedrockSecretKey,
bedrockRegion: routerConfig.bedrockRegion,
bedrockSession,
workflowId: ctx.workflowId,
workspaceId: ctx.workspaceId,
responseFormat: {
Expand Down Expand Up @@ -350,6 +321,7 @@ export class RouterBlockHandler implements BlockHandler {
providerId,
request: providerRequest,
resolvedSecretTraceRegistry: modelInputProjection.registry,
credentials: { inputs: routerConfig, callerLabel: 'vertex-router' },
})

let chosenRouteId: string
Expand Down
Loading
Loading