Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions aibridge/intercept/chatcompletions/paramswrap_internal_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,23 @@ func TestChatCompletionRequestBodyWithoutCacheControlIsUnchanged(t *testing.T) {
require.Equal(t, typed, body)
}

// chatd sends a PDF, from a user attachment or from a tool result, as a
// user file part, so the typed round trip must keep it.
func TestChatCompletionRequestBodyKeepsUserFilePart(t *testing.T) {
t.Parallel()

const filePart = `{"type":"file","file":{"file_data":"data:application/pdf;base64,JVBERi0xLjc=","filename":"report.pdf"}}`
var req ChatCompletionNewParamsWrapper
require.NoError(t, json.Unmarshal([]byte(`{"model":"gpt-4o","messages":[{"role":"user","content":[`+filePart+`]}]}`), &req))

body, err := (&interceptionBase{
req: &req,
cfg: intercept.Config{BaseURL: "https://api.openai.com/v1"},
}).chatCompletionRequestBody()
require.NoError(t, err)
require.JSONEq(t, `[`+filePart+`]`, gjson.GetBytes(body, "messages.0.content").Raw)
}

func TestPreservedCacheControlFieldsAreCapped(t *testing.T) {
t.Parallel()

Expand Down
1 change: 1 addition & 0 deletions coderd/x/chatd/chatloop/chatloop.go
Original file line number Diff line number Diff line change
Expand Up @@ -1542,6 +1542,7 @@ func (b *toolBatch) runOne(
Data: base64.StdEncoding.EncodeToString(resp.Data),
MediaType: resp.MediaType,
Text: text,
Filename: resp.Filename,
}
default:
result.Result = fantasy.ToolResultOutputContentText{
Expand Down
2 changes: 2 additions & 0 deletions coderd/x/chatd/chatloop/chatloop_run_internal_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1405,6 +1405,7 @@ func TestExecuteSingleTool_MediaBase64Encoding(t *testing.T) {
Type: "image",
Data: originalBytes,
MediaType: "image/jpeg",
Filename: "photo.jpg",
}, nil
},
)
Expand All @@ -1423,6 +1424,7 @@ func TestExecuteSingleTool_MediaBase64Encoding(t *testing.T) {
media, ok := result.Result.(fantasy.ToolResultOutputContentMedia)
require.True(t, ok, "expected ToolResultOutputContentMedia")
require.Equal(t, "image/jpeg", media.MediaType)
require.Equal(t, "photo.jpg", media.Filename)

decoded, err := base64.StdEncoding.DecodeString(media.Data)
require.NoError(t, err, "Data should be valid base64")
Expand Down
6 changes: 5 additions & 1 deletion coderd/x/chatd/chatprompt/chatprompt.go
Original file line number Diff line number Diff line change
Expand Up @@ -558,6 +558,7 @@ func toolResultContentToPart(
Data: output.Data,
MimeType: output.MediaType,
Text: strings.ToValidUTF8(output.Text, "\uFFFD"),
Filename: output.Filename,
}
// Tool renderers only receive the persisted result JSON, while
// ClientMetadata is consumed later to append sibling file parts.
Expand Down Expand Up @@ -1151,6 +1152,7 @@ func toolResultPartToMessagePart(logger slog.Logger, part codersdk.ChatMessagePa
Data: media.Data,
MediaType: media.MimeType,
Text: strings.ToValidUTF8(media.Text, "\uFFFD"),
Filename: media.Filename,
},
ProviderOptions: opts,
}
Expand Down Expand Up @@ -1207,7 +1209,8 @@ func toolResultPartToMessagePart(logger slog.Logger, part codersdk.ChatMessagePa
// cannot drift.
//
// The "mime_type" key intentionally diverges from the fantasy
// struct tag (json:"media_type"). Optional attachment identity
// struct tag (json:"media_type"). Filename is the tool's name for the
// media and reaches the model. Optional attachment identity
// fields are UI hints only. They let the frontend recognize when the
// same media was also promoted into a durable file part, but the prompt
// reconstruction path must continue to ignore them. Keep additions
Expand All @@ -1216,6 +1219,7 @@ type persistedMediaResult struct {
Data string `json:"data"`
MimeType string `json:"mime_type"`
Text string `json:"text"`
Filename string `json:"filename,omitempty"`
AttachmentFileID string `json:"attachment_file_id,omitempty"`
AttachmentName string `json:"attachment_name,omitempty"`
}
Expand Down
3 changes: 3 additions & 0 deletions coderd/x/chatd/chatprompt/chatprompt_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2257,6 +2257,7 @@ func TestMediaToolResultRoundTrip(t *testing.T) {
const callID = "call-screenshot-1"
const toolName = "computer"
const mimeType = "image/png"
const filename = "screenshot.png"

// Use PartFromContent (the production write path) to
// produce the SDK part, rather than hand-crafting JSON.
Expand All @@ -2269,6 +2270,7 @@ func TestMediaToolResultRoundTrip(t *testing.T) {
Result: fantasy.ToolResultOutputContentMedia{
Data: mediaImageData,
MediaType: mimeType,
Filename: filename,
},
})

Expand Down Expand Up @@ -2301,6 +2303,7 @@ func TestMediaToolResultRoundTrip(t *testing.T) {
require.True(t, ok, "expected ToolResultOutputContentMedia, got %T", resultPart.Output)
require.Equal(t, mediaImageData, mediaOutput.Data)
require.Equal(t, mimeType, mediaOutput.MediaType)
require.Equal(t, filename, mediaOutput.Filename)
}

func toolCallAndResultMessages(t *testing.T, result codersdk.ChatMessagePart) []database.ChatMessage {
Expand Down
6 changes: 3 additions & 3 deletions coderd/x/chatd/chatprovider/chatprovider.go
Original file line number Diff line number Diff line change
Expand Up @@ -155,13 +155,13 @@ func (m Model) AcceptsFilePartMediaType(mediaType string) bool {
case fantasyanthropic.Name:
return isImage || isText || isPDF
case fantasyopenai.Name:
// Chat Completions accepts text and audio as native file parts.
// Chat Completions also accepts audio as native file parts.
if m.transport.UsesResponses() {
return isImage || isPDF
}
return isImage || isText || isAudio || isPDF
return isImage || isAudio || isPDF
case fantasyopenaicompat.Name:
return isImage || isText || isAudio || isPDF
return isImage || isAudio || isPDF
default:
return false
}
Expand Down
134 changes: 134 additions & 0 deletions coderd/x/chatd/chatprovider/chatprovider_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -944,6 +944,140 @@ func TestModelFromConfig_AnthropicPDFFilePartReachesProvider(t *testing.T) {
_ = testutil.TryReceive(ctx, t, called)
}

// TestModelFromConfig_PDFToolResultReachesProvider checks that a PDF tool
// result reaches each transport in the native file shape that the transport
// uses for a user attachment, with the tool's filename.
func TestModelFromConfig_PDFToolResultReachesProvider(t *testing.T) {
t.Parallel()

const filename = "quarterly_report.v1.pdf"
pdfData := base64.StdEncoding.EncodeToString([]byte("%PDF-1.7\nfake pdf bytes for tool result test"))
dataURL := "data:application/pdf;base64," + pdfData

prompt := []fantasy.Message{
{
Role: fantasy.MessageRoleUser,
Content: []fantasy.MessagePart{fantasy.TextPart{Text: "Read the report."}},
},
{
Role: fantasy.MessageRoleAssistant,
Content: []fantasy.MessagePart{fantasy.ToolCallPart{ToolCallID: "call-1", ToolName: "read_report", Input: "{}"}},
},
{
Role: fantasy.MessageRoleTool,
Content: []fantasy.MessagePart{fantasy.ToolResultPart{
ToolCallID: "call-1",
Output: fantasy.ToolResultOutputContentMedia{
Data: pdfData,
MediaType: "application/pdf",
Filename: filename,
},
}},
},
}

// Chat Completions cannot put a file in a tool message, so the PDF goes
// in a user message right after the tool message.
chatCompletionsTail := []string{
`{"role":"tool","tool_call_id":"call-1","content":"The tool returned application/pdf content; see the following user message."}`,
`{"role":"user","content":[{"type":"file","file":{"file_data":"` + dataURL + `","filename":"` + filename + `"}}]}`,
}

for _, tc := range []struct {
name string
provider string
model string
config *codersdk.ChatModelCallConfig
// wantTail is the end of the prompt on the wire, so nothing may
// follow the item that carries the PDF.
wantTail []string
}{
{
name: "AnthropicDocumentInToolResult",
provider: fantasyanthropic.Name,
model: "claude-sonnet-4-20250514",
wantTail: []string{
`{"role":"user","content":[{"type":"tool_result","tool_use_id":"call-1","content":[` +
`{"type":"document","title":"quarterly report v1 pdf","source":{"type":"base64","media_type":"application/pdf","data":"` + pdfData + `"}}]}]}`,
},
},
{
name: "OpenAIChatCompletionsUserFilePart",
provider: fantasyopenai.Name,
model: "gpt-5",
config: &codersdk.ChatModelCallConfig{OpenAIConfig: &codersdk.ChatModelOpenAIConfig{UseResponsesAPI: new(false)}},
wantTail: chatCompletionsTail,
},
{
name: "OpenAIResponsesFileInFunctionCallOutput",
provider: fantasyopenai.Name,
model: "gpt-5",
config: &codersdk.ChatModelCallConfig{OpenAIConfig: &codersdk.ChatModelOpenAIConfig{UseResponsesAPI: new(true)}},
wantTail: []string{
`{"type":"function_call_output","call_id":"call-1","output":[{"type":"input_file","file_data":"` + dataURL + `","filename":"` + filename + `"}]}`,
},
},
{
name: "OpenAICompatUserFilePart",
provider: fantasyopenaicompat.Name,
model: "test-model",
wantTail: chatCompletionsTail,
},
} {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitShort)

// Each request sends its prompt items: Anthropic messages, Chat
// Completions messages, or Responses input items.
items := make(chan []json.RawMessage, 1)
var serverURL string
if tc.provider == fantasyanthropic.Name {
serverURL = chattest.NewAnthropic(t, func(req *chattest.AnthropicRequest) chattest.AnthropicResponse {
messages := make([]json.RawMessage, 0, len(req.Messages))
for _, message := range req.Messages {
raw, err := json.Marshal(message)
assert.NoError(t, err)
messages = append(messages, raw)
}
items <- messages
return chattest.AnthropicNonStreamingResponse("ok")
})
} else {
serverURL = chattest.NewOpenAI(t, func(req *chattest.OpenAIRequest) chattest.OpenAIResponse {
var body struct {
Messages []json.RawMessage `json:"messages"`
Input []json.RawMessage `json:"input"`
}
assert.NoError(t, json.Unmarshal(req.RawBody, &body))
items <- append(body.Messages, body.Input...)
return chattest.OpenAINonStreamingResponse("ok")
})
}

model, err := chatprovider.ModelFromConfig(
tc.provider, tc.model,
chatprovider.ProviderAPIKeys{
ByProvider: map[string]string{tc.provider: "test-key"},
BaseURLByProvider: map[string]string{tc.provider: serverURL},
},
chatprovider.UserAgent(), nil, nil, tc.config,
)
require.NoError(t, err)

_, err = model.LanguageModel().Generate(ctx, fantasy.Call{Prompt: prompt})
require.NoError(t, err)

got := testutil.TryReceive(ctx, t, items)
require.GreaterOrEqual(t, len(got), len(tc.wantTail))
gotTail := got[len(got)-len(tc.wantTail):]
for i, want := range tc.wantTail {
require.JSONEq(t, want, string(gotTail[i]))
}
})
}
}

func TestModelFromConfig_NilExtraHeaders(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitShort)
Expand Down
18 changes: 9 additions & 9 deletions coderd/x/chatd/chatprovider/file_parts_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ func TestAcceptsFilePartMediaType(t *testing.T) {
t.Parallel()

// A representative Responses-capable model ID. Non-Responses models
// use the Chat Completions path, which accepts text/* natively.
// use the Chat Completions path, which also accepts audio.
const responsesModel = "gpt-4o"
const nonResponsesModel = "babbage-002"

Expand All @@ -74,9 +74,10 @@ func TestAcceptsFilePartMediaType(t *testing.T) {
{"openai-image", "openai", responsesModel, "image/png", true, nil},
{"openai-pdf", "openai", responsesModel, "application/pdf", true, nil},

// OpenAI Chat Completions (non-Responses models) accepts text/*
// and audio as native file parts.
{"openai-non-responses-text", "openai", nonResponsesModel, "text/plain", true, nil},
// OpenAI Chat Completions (non-Responses models) accepts audio as
// native file parts, but not text files.
{"openai-non-responses-text", "openai", nonResponsesModel, "text/plain", false, nil},
{"openai-non-responses-audio", "openai", nonResponsesModel, "audio/mpeg", true, nil},
{"openai-non-responses-json", "openai", nonResponsesModel, "application/json", false, nil},
{"openai-non-responses-image", "openai", nonResponsesModel, "image/png", true, nil},

Expand All @@ -86,8 +87,8 @@ func TestAcceptsFilePartMediaType(t *testing.T) {
{"anthropic-pdf", "anthropic", "", "application/pdf", true, nil},
{"anthropic-image", "anthropic", "", "image/webp", true, nil},

// OpenAI-compatible accepts text/*, images, audio, and PDFs.
{"openaicompat-text", "openai-compat", "", "text/plain", true, nil},
// OpenAI-compatible accepts images, audio, and PDFs.
{"openaicompat-text", "openai-compat", "", "text/plain", false, nil},
{"openaicompat-json", "openai-compat", "", "application/json", false, nil},
{"openaicompat-audio", "openai-compat", "", "audio/mpeg", true, nil},

Expand All @@ -105,13 +106,12 @@ func TestAcceptsFilePartMediaType(t *testing.T) {
// Provider name normalization is case-insensitive.
{"anthropic-uppercase", "Anthropic", "", "text/plain", true, nil},

{"openai-forced-responses-text", "openai", nonResponsesModel, "text/plain", false, &forceResponses},
{"openai-forced-responses-audio", "openai", nonResponsesModel, "audio/mpeg", false, &forceResponses},
{"openai-forced-responses-image", "openai", nonResponsesModel, "image/png", true, &forceResponses},
{"openai-forced-completions-text", "openai", responsesModel, "text/plain", true, &forceCompletions},
{"openai-forced-completions-audio", "openai", responsesModel, "audio/mpeg", true, &forceCompletions},

{"anthropic-ignores-override", "anthropic", "", "text/plain", true, &forceResponses},
{"openaicompat-ignores-override", "openai-compat", "", "text/plain", true, &forceResponses},
{"openaicompat-ignores-override", "openai-compat", "", "audio/mpeg", true, &forceResponses},
}

for _, tc := range cases {
Expand Down
26 changes: 13 additions & 13 deletions coderd/x/chatd/chatprovider/responses_api_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -125,12 +125,12 @@ func TestModelTransportConsumersAgree(t *testing.T) {
serviceTier := "auto"

cases := []struct {
name string
modelID string
override *bool
wantPath string
wantOptions fantasy.ProviderOptionsData
wantAcceptText bool
name string
modelID string
override *bool
wantPath string
wantOptions fantasy.ProviderOptionsData
wantAcceptAudio bool
}{
{
name: "ForceResponsesOnUnknownModel",
Expand All @@ -140,12 +140,12 @@ func TestModelTransportConsumersAgree(t *testing.T) {
wantOptions: &fantasyopenai.ResponsesProviderOptions{},
},
{
name: "ForceCompletionsOnKnownModel",
modelID: responsesModel,
override: &forceCompletions,
wantPath: "/chat/completions",
wantOptions: &fantasyopenai.ProviderOptions{},
wantAcceptText: true,
name: "ForceCompletionsOnKnownModel",
modelID: responsesModel,
override: &forceCompletions,
wantPath: "/chat/completions",
wantOptions: &fantasyopenai.ProviderOptions{},
wantAcceptAudio: true,
},
{
name: "UnsetFollowsKnownModelList",
Expand Down Expand Up @@ -211,7 +211,7 @@ func TestModelTransportConsumersAgree(t *testing.T) {
}, nil)
require.IsType(t, tc.wantOptions, effortOptions[fantasyopenai.Name])

require.Equal(t, tc.wantAcceptText, model.AcceptsFilePartMediaType("text/plain"))
require.Equal(t, tc.wantAcceptAudio, model.AcceptsFilePartMediaType("audio/mpeg"))
require.True(t, model.AcceptsFilePartMediaType("image/png"))
})
}
Expand Down
6 changes: 6 additions & 0 deletions coderd/x/chatd/mcpclient/inline_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -289,6 +289,11 @@ func TestConnectInline_RedactsSensitiveValues(t *testing.T) {
handler: func(context.Context, *mcp.CallToolRequest) (*mcp.CallToolResult, error) {
result := textToolResult("result mentions " + secret + " and " + serverURL + " and " + bearerToken + " and " + pathToken + " and " + cookieToken + " and " + signatureToken)
result.StructuredContent = map[string]any{"token": secret, "url": serverURL}
// The resource filename is the last URI path segment, here the
// path token.
result.Content = append(result.Content, &mcp.EmbeddedResource{Resource: &mcp.ResourceContents{
URI: serverURL, MIMEType: "application/pdf", Blob: []byte("pdf-data"),
}})
return result, nil
},
}
Expand Down Expand Up @@ -369,6 +374,7 @@ func TestConnectInline_RedactsSensitiveValues(t *testing.T) {
require.NotContains(t, resp.Content, cookieToken)
require.NotContains(t, resp.Content, signatureToken)
require.Contains(t, resp.Content, `"token":"[REDACTED]"`)
require.Equal(t, "[REDACTED].pdf", resp.Filename)

resp, err = failingTool.Run(ctx, fantasy.ToolCall{ID: "call-2", Input: "{}"})
require.NoError(t, err)
Expand Down
Loading
Loading