Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions apps/docs/content/docs/en/workflows/blocks/api.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ The request payload for POST, PUT, and PATCH, sent as JSON. Type it directly, or
- **Retries.** Number of retry attempts on timeouts, `429` responses, and `5xx` errors. Defaults to 0.
- **Retry delay / Max retry delay (ms).** The exponential-backoff bounds used between retries.
- **Retry non-idempotent methods.** Off by default, so POST and PATCH are not retried, which avoids duplicate writes. Turn it on only when a repeated request is safe.
- **Proxy URL.** Optional `http://` proxy the request egresses through, such as a residential proxy for targets that block datacenter IPs. The proxy host must be publicly reachable, and only the `http://` scheme is supported (an `https://` target still tunnels through it securely). Keep proxy credentials in an environment variable and reference it as `{{PROXY_URL}}` rather than typing them into the field.

## Outputs

Expand Down
13 changes: 13 additions & 0 deletions apps/sim/blocks/blocks/api.ts
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,15 @@ Example:
description: 'Allow retries for POST/PATCH requests (may create duplicate requests)',
mode: 'advanced',
},
{
id: 'proxyUrl',
title: 'Proxy URL',
type: 'short-input',
placeholder: 'http://user:pass@proxy.host:port or {{PROXY_URL}}',
description:
'Optional. Route this request through an http:// proxy (e.g. a residential proxy). Must be http://; the proxy host must be publicly reachable. Keep credentials in an environment variable and reference it like {{PROXY_URL}}.',
mode: 'advanced',
},
],
tools: {
access: ['http_request'],
Expand All @@ -144,6 +153,10 @@ Example:
type: 'boolean',
description: 'Allow retries for non-idempotent methods like POST/PATCH',
},
proxyUrl: {
type: 'string',
description: 'Optional http:// proxy URL to route the request through',
},
},
outputs: {
data: { type: 'json', description: 'API response data (JSON, text, or other formats)' },
Expand Down
89 changes: 84 additions & 5 deletions apps/sim/lib/core/security/input-validation.server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ import type { LookupFunction } from 'net'
import { createLogger } from '@sim/logger'
import { toError } from '@sim/utils/errors'
import { omit } from '@sim/utils/object'
import { HttpProxyAgent } from 'http-proxy-agent'
import { HttpsProxyAgent } from 'https-proxy-agent'
import * as ipaddr from 'ipaddr.js'
import { Agent, type RequestInit as UndiciRequestInit, fetch as undiciFetch } from 'undici'
import { isHosted, isPrivateDatabaseHostsAllowed } from '@/lib/core/config/env-flags'
Expand Down Expand Up @@ -148,6 +150,71 @@ export async function validateUrlWithDNS(
}
}

/**
* Result of validating a user-supplied HTTP proxy URL.
*/
export interface ProxyValidationResult {
isValid: boolean
/** Proxy URL with hostname rewritten to the resolved IP (creds/port preserved) to pin the proxy connection. */
pinnedProxyUrl?: string
error?: string
}

/**
* Validates a user-supplied HTTP proxy URL and returns an IP-pinned form.
*
* When a request routes through a proxy, the TCP connection targets the proxy
* host (the proxy resolves the destination), so target-IP pinning no longer
* governs egress and the proxy URL becomes the SSRF surface. This function:
* 1. Enforces the `http:` scheme (raw TCP to the proxy, no TLS-to-proxy SNI to
* reconcile, so the host can be safely rewritten to an IP).
* 2. Resolves the proxy host's DNS and blocks private/reserved/loopback IPs via
* {@link validateUrlWithDNS}.
* 3. Pins the connection by rewriting the hostname to the resolved IP while
* preserving credentials/port, closing the DNS-rebinding (TOCTOU) window.
*
* @param proxyUrl - The proxy URL (e.g. `http://user:pass@host:port`)
*/
export async function validateAndPinProxyUrl(
proxyUrl: string | null | undefined
): Promise<ProxyValidationResult> {
if (!proxyUrl || typeof proxyUrl !== 'string') {
return { isValid: false, error: 'proxyUrl must be a string' }
}

let parsed: URL
try {
parsed = new URL(proxyUrl)
} catch {
return { isValid: false, error: 'proxyUrl must be a valid URL' }
}

if (parsed.protocol !== 'http:') {
return {
isValid: false,
error: 'proxyUrl must use http:// (https/socks proxies are not supported)',
}
}

const validation = await validateUrlWithDNS(proxyUrl, 'proxyUrl', { allowHttp: true })
if (!validation.isValid) {
return { isValid: false, error: validation.error }
}

const resolvedIP = validation.resolvedIP!

// validateUrlWithDNS permits loopback for self-hosted dev targets; a proxy governs
// egress, so loopback/private proxy hosts stay blocked unconditionally.
if (isPrivateOrReservedIP(resolvedIP)) {
return { isValid: false, error: 'proxyUrl resolves to a blocked IP address' }
}

// Bracket IPv6 literals: assigning an unbracketed IPv6 address to URL.hostname
// is a no-op, which would leave the DNS hostname in place and reopen rebinding.
parsed.hostname = resolvedIP.includes(':') ? `[${resolvedIP}]` : resolvedIP
return { isValid: true, pinnedProxyUrl: parsed.toString() }
Comment thread
waleedlatif1 marked this conversation as resolved.
Comment thread
waleedlatif1 marked this conversation as resolved.
}

/**
* Validates a database hostname by resolving DNS and checking the resolved IP
* against private/reserved ranges to prevent SSRF via database connections.
Expand Down Expand Up @@ -343,6 +410,12 @@ export interface SecureFetchOptions {
signal?: AbortSignal
/** Drop the Authorization header when following a redirect, so it is not sent to the redirect target's origin. */
stripAuthOnRedirect?: boolean
/**
* Pre-validated, IP-pinned `http://` proxy URL (see {@link validateAndPinProxyUrl}).
* When set, the connection routes through this proxy and target-IP pinning is
* bypassed (the proxy resolves the target).
*/
proxyUrl?: string
}

export class SecureFetchHeaders {
Expand Down Expand Up @@ -678,11 +751,17 @@ export async function secureFetchWithPinnedIP(
const defaultPort = isHttps ? 443 : 80
const port = parsed.port ? Number.parseInt(parsed.port, 10) : defaultPort

const lookup = createPinnedLookup(resolvedIP)

const agentOptions: http.AgentOptions = { lookup }

const agent = isHttps ? new https.Agent(agentOptions) : new http.Agent(agentOptions)
let agent: http.Agent
if (options.proxyUrl) {
// Proxy connection is already IP-pinned by validateAndPinProxyUrl; target-IP
// pinning is intentionally bypassed (the proxy resolves the target). https
// targets tunnel via CONNECT, http targets use absolute-URI forwarding.
agent = isHttps ? new HttpsProxyAgent(options.proxyUrl) : new HttpProxyAgent(options.proxyUrl)
} else {
const lookup = createPinnedLookup(resolvedIP)
const agentOptions: http.AgentOptions = { lookup }
agent = isHttps ? new https.Agent(agentOptions) : new http.Agent(agentOptions)
}

const { 'accept-encoding': _, ...sanitizedHeaders } = options.headers ?? {}

Expand Down
68 changes: 68 additions & 0 deletions apps/sim/lib/core/security/input-validation.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import {
} from '@/lib/core/security/input-validation'
import {
isPrivateOrReservedIP,
validateAndPinProxyUrl,
validateDatabaseHost,
validateUrlWithDNS,
} from '@/lib/core/security/input-validation.server'
Expand Down Expand Up @@ -843,6 +844,73 @@ describe('validateDatabaseHost', () => {
})
})

describe('validateAndPinProxyUrl', () => {
it('should reject a null/empty proxy URL', async () => {
expect((await validateAndPinProxyUrl(null)).isValid).toBe(false)
expect((await validateAndPinProxyUrl('')).isValid).toBe(false)
})

it('should reject a malformed URL', async () => {
const result = await validateAndPinProxyUrl('not a url')
expect(result.isValid).toBe(false)
expect(result.error).toContain('valid URL')
})

it('should reject an https:// proxy scheme', async () => {
const result = await validateAndPinProxyUrl('https://proxy.example.com:8080')
expect(result.isValid).toBe(false)
expect(result.error).toContain('http://')
})

it('should reject a socks5:// proxy scheme', async () => {
const result = await validateAndPinProxyUrl('socks5://proxy.example.com:1080')
expect(result.isValid).toBe(false)
expect(result.error).toContain('http://')
})

it('should reject a proxy host that is a private IP', async () => {
const result = await validateAndPinProxyUrl('http://user:pass@192.168.1.1:8080')
expect(result.isValid).toBe(false)
expect(result.error).toMatch(/private IP|blocked IP/)
})

it('should reject a loopback proxy host even off the hosted platform', async () => {
const localhost = await validateAndPinProxyUrl('http://localhost:3128')
expect(localhost.isValid).toBe(false)
expect(localhost.error).toContain('blocked IP')
const loopback = await validateAndPinProxyUrl('http://127.0.0.1:3128')
expect(loopback.isValid).toBe(false)
expect(loopback.error).toContain('blocked IP')
})

it('should reject a proxy host that is the metadata IP', async () => {
const result = await validateAndPinProxyUrl('http://169.254.169.254:80')
expect(result.isValid).toBe(false)
expect(result.error).toMatch(/private IP|blocked IP/)
})

it('should accept a public proxy host and pin the hostname to the resolved IP, preserving creds/port', async () => {
const result = await validateAndPinProxyUrl('http://user:pass@8.8.8.8:8080')
expect(result.isValid).toBe(true)
const pinned = new URL(result.pinnedProxyUrl!)
expect(pinned.protocol).toBe('http:')
expect(pinned.hostname).toBe('8.8.8.8')
expect(pinned.username).toBe('user')
expect(pinned.password).toBe('pass')
expect(pinned.port).toBe('8080')
})

it('should bracket an IPv6 resolved address so the pinned host is the IP, not the original name', async () => {
const result = await validateAndPinProxyUrl('http://user:pass@[2606:4700:4700::1111]:8080')
expect(result.isValid).toBe(true)
const pinned = new URL(result.pinnedProxyUrl!)
expect(pinned.hostname).toBe('[2606:4700:4700::1111]')
expect(pinned.username).toBe('user')
expect(pinned.password).toBe('pass')
expect(pinned.port).toBe('8080')
})
})

describe('validateInteger', () => {
describe('valid integers', () => {
it.concurrent('should accept positive integers', () => {
Expand Down
2 changes: 2 additions & 0 deletions apps/sim/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,8 @@
"gray-matter": "^4.0.3",
"groq-sdk": "^0.15.0",
"html-to-text": "^9.0.5",
"http-proxy-agent": "7.0.2",
"https-proxy-agent": "7.0.6",
"idb-keyval": "6.2.2",
"image-size": "1.2.1",
"imapflow": "1.2.4",
Expand Down
5 changes: 5 additions & 0 deletions apps/sim/tools/http/request.ts
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,11 @@ export const requestTool: ToolConfig<RequestParams, RequestResponse> = {
visibility: 'user-or-llm',
description: 'Form data to send (will set appropriate Content-Type)',
},
proxyUrl: {
type: 'string',
visibility: 'user-only',
description: 'Route the request through an http:// proxy (e.g. http://user:pass@host:port).',
},
timeout: {
type: 'number',
visibility: 'user-only',
Expand Down
1 change: 1 addition & 0 deletions apps/sim/tools/http/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ export interface RequestParams {
params?: TableRow[] | string
pathParams?: Record<string, string>
formData?: Record<string, string | Blob>
proxyUrl?: string
timeout?: number
retries?: number
retryDelayMs?: number
Expand Down
71 changes: 71 additions & 0 deletions apps/sim/tools/index.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -958,6 +958,77 @@ describe('Automatic Internal Route Detection', () => {
Object.assign(tools, originalTools)
})

it('should validate + pin a proxyUrl param and pass it to secureFetchWithPinnedIP', async () => {
inputValidationMockFns.mockValidateAndPinProxyUrl.mockResolvedValue({
isValid: true,
pinnedProxyUrl: 'http://user:pass@1.2.3.4:8080/',
})

const mockTool = {
id: 'test_external_proxy',
name: 'Test External Proxy Tool',
description: 'A test tool that routes through a proxy',
version: '1.0.0',
params: {},
request: {
url: 'https://api.example.com/endpoint',
method: 'GET',
headers: () => ({ 'Content-Type': 'application/json' }),
},
transformResponse: vi.fn().mockResolvedValue({ success: true, output: {} }),
}

const originalTools = { ...tools }
;(tools as any).test_external_proxy = mockTool

await executeTool('test_external_proxy', { proxyUrl: 'http://user:pass@proxy.host:8080' })

expect(inputValidationMockFns.mockValidateAndPinProxyUrl).toHaveBeenCalledWith(
'http://user:pass@proxy.host:8080'
)
expect(mockSecureFetchWithPinnedIP).toHaveBeenCalledWith(
'https://api.example.com/endpoint',
'93.184.216.34',
expect.objectContaining({ proxyUrl: 'http://user:pass@1.2.3.4:8080/' })
)

Object.assign(tools, originalTools)
})

it('should throw when the proxyUrl param fails validation', async () => {
inputValidationMockFns.mockValidateAndPinProxyUrl.mockResolvedValue({
isValid: false,
error: 'proxyUrl must use http:// (https/socks proxies are not supported)',
})

const mockTool = {
id: 'test_external_bad_proxy',
name: 'Test External Bad Proxy Tool',
description: 'A test tool with an invalid proxy',
version: '1.0.0',
params: {},
request: {
url: 'https://api.example.com/endpoint',
method: 'GET',
headers: () => ({ 'Content-Type': 'application/json' }),
},
transformResponse: vi.fn().mockResolvedValue({ success: true, output: {} }),
}

const originalTools = { ...tools }
;(tools as any).test_external_bad_proxy = mockTool

const result = await executeTool('test_external_bad_proxy', {
proxyUrl: 'https://proxy.host:8080',
})

expect(result.success).toBe(false)
expect(result.error).toContain('Invalid proxy URL')
expect(mockSecureFetchWithPinnedIP).not.toHaveBeenCalled()

Object.assign(tools, originalTools)
})

it('should handle dynamic URLs that resolve to internal routes', async () => {
const mockTool = {
id: 'test_dynamic_internal',
Expand Down
11 changes: 11 additions & 0 deletions apps/sim/tools/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import { DEFAULT_EXECUTION_TIMEOUT_MS, getMaxExecutionTimeout } from '@/lib/core
import { getHostedKeyRateLimiter } from '@/lib/core/rate-limiter'
import {
secureFetchWithPinnedIP,
validateAndPinProxyUrl,
validateUrlWithDNS,
} from '@/lib/core/security/input-validation.server'
import { PlatformEvents } from '@/lib/core/telemetry'
Expand Down Expand Up @@ -1809,13 +1810,23 @@ async function executeToolRequest(
throw new Error(`Invalid tool URL: ${urlValidation.error}`)
}

let proxyOption: string | undefined
if (requestParams.proxyUrl) {
const proxyValidation = await validateAndPinProxyUrl(requestParams.proxyUrl)
if (!proxyValidation.isValid) {
throw new Error(`Invalid proxy URL: ${proxyValidation.error}`)
}
proxyOption = proxyValidation.pinnedProxyUrl
}

const secureResponse = await secureFetchWithPinnedIP(fullUrl, urlValidation.resolvedIP!, {
method: requestParams.method,
headers: headersRecord,
body: requestParams.body ?? undefined,
timeout: requestParams.timeout,
maxResponseBytes: MAX_TOOL_RESPONSE_BODY_BYTES,
signal,
proxyUrl: proxyOption,
})

const responseHeaders = new Headers(secureResponse.headers.toRecord())
Expand Down
11 changes: 11 additions & 0 deletions apps/sim/tools/utils.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -209,6 +209,17 @@ describe('formatRequestParams', () => {

expect(result.body).toBe('{"prompt": "Hello"}\n{"prompt": "World"}')
})

it.concurrent('should pass through a non-empty proxyUrl (trimmed)', () => {
const result = formatRequestParams(mockTool, { proxyUrl: ' http://user:pass@host:8080 ' })
expect(result.proxyUrl).toBe('http://user:pass@host:8080')
})

it.concurrent('should omit proxyUrl when blank, whitespace, or absent', () => {
expect(formatRequestParams(mockTool, {}).proxyUrl).toBeUndefined()
expect(formatRequestParams(mockTool, { proxyUrl: '' }).proxyUrl).toBeUndefined()
expect(formatRequestParams(mockTool, { proxyUrl: ' ' }).proxyUrl).toBeUndefined()
})
})

describe('validateRequiredParametersAfterMerge', () => {
Expand Down
Loading
Loading