Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions coderd/x/chatd/ARCHITECTURE.md
Original file line number Diff line number Diff line change
Expand Up @@ -892,6 +892,8 @@ When receiving streaming message parts from the LLM API, the generation goroutin

Since the runner doesn't wait for goroutines to finish when it cancels them, and spawns new goroutines to perform new work immediately, the runner does not guarantee that any interrupted tool calls are fully stopped before continuing. Tool call interrupts are best-effort.

TODO (CODAGT-757): for a capable workspace agent, the interrupt goroutine now runs the interrupt run before `FinishInterruption`: it dispatches every unresolved `execute` call through `chatloop.ExecuteLocalTools` with the interrupt cause, so the tool cancels the tool call on the agent, which kills any process it started, then sends its start again and commits the real result (canceled with partial output, completed, not run, or unknown). A cancel that fails is never followed by the start. For any other agent, unresolved calls keep the generic interrupted result. Rewrite this paragraph and the interrupt goroutine steps.

Tool calls have at least once semantics: if the goroutine executes a tool call, and the replica crashes before the result is persisted, another replica will execute the tool call again later. Future work may include adding a mechanism to ensure at most once semantics.

TODO (CODAGT-757): for a workspace agent with API version 2.13 or later, checked once per turn, `execute` sends tool call headers with its start request, so the agent starts at most one process per tool call and a task retry or ownership change gets the same process and result instead of starting another. The agent owns the execute deadline, and a request the agent does not answer is sent again for up to one minute before the result says the outcome is unknown. Rewrite this paragraph.
Expand Down
133 changes: 133 additions & 0 deletions coderd/x/chatd/chatd_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -3579,6 +3579,139 @@ func TestActiveServer_InterruptionBehavior(t *testing.T) {
require.False(t, result.CreatedAt.Before(*call.CreatedAt))
})

t.Run("interrupt run cancels a foreground execute and commits its result", func(t *testing.T) {
t.Parallel()

ctx := testutil.Context(t, testutil.WaitLong)
db, ps := dbtestutil.NewDB(t)
var requestCount atomic.Int32
openAIURL := chattest.NewOpenAI(t, func(req *chattest.OpenAIRequest) chattest.OpenAIResponse {
if !req.Stream {
return chattest.OpenAINonStreamingResponse("title")
}
if requestCount.Add(1) == 1 {
chunk := chattest.OpenAIToolCallChunk("execute", `{"command":"make test","timeout":"10m"}`)
chunk.Choices[0].ToolCalls[0].ID = "tc-exec"
return chattest.OpenAIStreamingResponse(chunk)
}
return chattest.OpenAIStreamingResponse(chattest.OpenAITextChunks("after interrupt")...)
})
user, org, model := seedChatDependenciesWithProvider(t, db, "openai-compat", openAIURL)
ws, dbAgent := seedWorkspaceWithAgent(t, db, user.ID)
require.NoError(t, db.UpdateWorkspaceAgentStartupByID(ctx, database.UpdateWorkspaceAgentStartupByIDParams{
ID: dbAgent.ID,
Version: "v2.99.0",
ExpandedDirectory: "/home/coder/project",
APIVersion: "2.13",
}))

mockConn := agentconnmock.NewMockAgentConn(gomock.NewController(t))
// Registered first so it matches before the helper's catch-all.
var chatIDHeader atomic.Value
mockConn.EXPECT().SetExtraHeaders(gomock.Any()).
Do(func(h http.Header) { chatIDHeader.Store(h.Get(workspacesdk.CoderChatIDHeader)) }).AnyTimes()
setupToolExecutionAgentConn(t, mockConn)
// requests records the agent requests for the tool call in order.
var mu sync.Mutex
var requests []string
record := func(ctx context.Context, request string) {
tc, ok := workspacesdk.ToolCallFromContext(ctx)
if !ok || tc.ID != "tc-exec" || tc.Name != "execute" {
t.Errorf("%s request tool call = %+v (set %v), want execute tool call tc-exec", request, tc, ok)
}
mu.Lock()
requests = append(requests, request)
mu.Unlock()
}
var processID atomic.Value
canceled := make(chan struct{})
toolWaiting := make(chan struct{})
var waitOnce sync.Once
mockConn.EXPECT().StartProcess(gomock.Any(), gomock.Any()).
DoAndReturn(func(ctx context.Context, req workspacesdk.StartProcessRequest) (workspacesdk.StartProcessResponse, error) {
record(ctx, "start")
if req.TimeoutMs != (10 * time.Minute).Milliseconds() {
t.Errorf("start timeout_ms = %d, want 10m", req.TimeoutMs)
}
tc, _ := workspacesdk.ToolCallFromContext(ctx)
chatID, err := uuid.Parse(fmt.Sprint(chatIDHeader.Load()))
if err != nil {
return workspacesdk.StartProcessResponse{}, xerrors.Errorf("start request without chat ID: %w", err)
}
id := workspacesdk.ToolCallUUID(chatID, tc.MessageID, tc.Name, tc.ID).String()
processID.Store(id)
return workspacesdk.StartProcessResponse{ID: id, Started: true}, nil
}).Times(2)
mockConn.EXPECT().ProcessOutput(gomock.Any(), gomock.Any(), gomock.Any()).
DoAndReturn(func(ctx context.Context, id string, _ *workspacesdk.ProcessOutputOptions) (workspacesdk.ProcessOutputResponse, error) {
if id != processID.Load() {
t.Errorf("output requested for process %q, want %v", id, processID.Load())
}
select {
case <-canceled:
exitCode := -1
return workspacesdk.ProcessOutputResponse{Canceled: true, Output: "partial output", ExitCode: &exitCode, DurationMs: 1500}, nil
default:
}
waitOnce.Do(func() { close(toolWaiting) })
<-ctx.Done()
return workspacesdk.ProcessOutputResponse{}, ctx.Err()
}).AnyTimes()
mockConn.EXPECT().CancelToolCall(gomock.Any(), gomock.Any()).
DoAndReturn(func(ctx context.Context, id string) error {
record(ctx, "cancel")
if id != processID.Load() {
t.Errorf("cancel for %q, want %v", id, processID.Load())
}
close(canceled)
return nil
}).Times(1)

server := newActiveTestServer(t, db, ps, func(cfg *chatd.Config) {
cfg.AIBridgeTransportFactory = chatAIGatewayTransportFactoryPointer(chattest.NewMockAIBridgeTransport(t, openAIURL))
cfg.AgentConn = func(_ context.Context, agentID uuid.UUID) (workspacesdk.AgentConn, func(), error) {
require.Equal(t, dbAgent.ID, agentID)
return mockConn, func() {}, nil
}
})
chat, err := server.CreateChat(ctx, chatd.CreateOptions{
OrganizationID: org.ID,
OwnerID: user.ID,
WorkspaceID: uuid.NullUUID{UUID: ws.ID, Valid: true},
AgentID: uuid.NullUUID{UUID: dbAgent.ID, Valid: true},
Title: "interrupt-run-execute",
ModelConfigID: model.ID,
InitialUserContent: []codersdk.ChatMessagePart{
codersdk.ChatMessageText("run the tests"),
},
})
require.NoError(t, err)

testutil.TryReceive(ctx, t, toolWaiting)
queued, err := server.SendMessage(ctx, chatd.SendMessageOptions{
ChatID: chat.ID,
CreatedBy: user.ID,
ModelConfigID: model.ID,
Content: []codersdk.ChatMessagePart{codersdk.ChatMessageText("stop")},
BusyBehavior: chatd.SendMessageBusyBehaviorInterrupt,
})
require.NoError(t, err)
require.True(t, queued.Queued)
waitForChatStatus(ctx, t, db, chat.ID, database.ChatStatusWaiting)

mu.Lock()
require.Equal(t, []string{"start", "cancel", "start"}, requests)
mu.Unlock()
result := requireToolResultPart(t, chatToolParts(ctx, t, db, chat.ID), "execute")
require.Equal(t, "tc-exec", result.ToolCallID)
require.False(t, result.IsError)
var executeResult chattool.ExecuteResult
require.NoError(t, json.Unmarshal(result.Result, &executeResult), string(result.Result))
require.Equal(t, "partial output", executeResult.Output)
require.Equal(t, -1, executeResult.ExitCode)
require.Contains(t, executeResult.Error, "canceled by the user after 1.5s")
})

t.Run("anthropic provider-only interruption commits no synthetic result", func(t *testing.T) {
t.Parallel()

Expand Down
4 changes: 4 additions & 0 deletions coderd/x/chatd/chattool/chattool.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,10 @@ const templateNotAvailableMessage = "template not available for chat workspaces;
// chat's workspace so the model recovers instead of concluding it is blocked.
const WorkspaceUnavailableHint = "The workspace is probably gone; use the create_workspace tool to make a new one"

// InterruptedToolResultMessage is the error result committed for an
// unresolved tool call when an interrupt cannot report its outcome.
const InterruptedToolResultMessage = "tool call was interrupted before it produced a result"

func workspaceLoadErrorResponse(err error) fantasy.ToolResponse {
return fantasy.NewTextErrorResponse(
xerrors.Errorf("load workspace: %w. %s", err, WorkspaceUnavailableHint).Error(),
Expand Down
118 changes: 104 additions & 14 deletions coderd/x/chatd/chattool/execute.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,17 @@ package chattool
import (
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"regexp"
"strings"
"time"

"charm.land/fantasy"
"golang.org/x/xerrors"

"github.com/coder/coder/v2/codersdk"
"github.com/coder/coder/v2/codersdk/workspacesdk"
"github.com/coder/quartz"
)
Expand Down Expand Up @@ -193,6 +196,14 @@ func executeTool(
}

if id, ok := ToolCallIdentityFromContext(ctx); ok {
if id.Cause == ToolCallCauseInterrupt {
// After the cancel the agent never starts the command, so
// the start below replays the recorded response or gets
// tool_call_canceled.
if err := cancelToolCall(ctx, conn, options.Clock, id); err != nil {
return cancelErrorResult(id, err)
}
}
if background {
return executeBackgroundToolCall(ctx, conn, options.Clock, id, args.Command, workDir, env)
}
Expand Down Expand Up @@ -231,6 +242,33 @@ func executeWords(id ToolCallIdentity) ToolCallWords {
}
}

// cancelToolCall cancels the tool call id on the agent until the agent
// answers or AgentAnswerTimeout ends. A nil error means the tool call's
// outcome is final: any process it started has stopped.
func cancelToolCall(ctx context.Context, conn workspacesdk.AgentConn, clock quartz.Clock, id ToolCallIdentity) error {
ctx = workspacesdk.WithToolCall(ctx, id.AgentToolCall())
return RequestUntilAnswered(ctx, clock, func(ctx context.Context) error {
return conn.CancelToolCall(ctx, id.UUID())
})
}

// cancelErrorResult converts the error of a failed cancel into a
// result. The start is never sent after a failed cancel, because it
// could start the command the user interrupted.
func cancelErrorResult(id ToolCallIdentity, err error) fantasy.ToolResponse {
var sdkErr *codersdk.Error
if errors.As(err, &sdkErr) && sdkErr.StatusCode() == http.StatusNotFound {
// The agent has no cancel route, so it is older than its
// api_version said and the call keeps the generic result.
return fantasy.NewTextErrorResponse(InterruptedToolResultMessage)
}
words := executeWords(id)
if text, ok := ToolCallErrorText(err, words); ok {
return errorResult(text)
}
return errorResult(UnknownOutcome(fmt.Sprintf("the workspace agent refused to cancel it (%v)", err), words.Effect, words.Check))
}

// startToolCallProcess sends req with the tool call headers of id until
// the agent answers or AgentAnswerTimeout ends.
func startToolCallProcess(
Expand Down Expand Up @@ -277,6 +315,9 @@ func executeBackgroundToolCall(
if err != nil {
return toolCallStartErrorResult(id, "start background process", err)
}
if id.Cause == ToolCallCauseInterrupt {
return marshalResult(readCanceledBackgroundProcess(ctx, conn, clock, resp.ID))
}
return marshalResult(ExecuteResult{
Success: true,
BackgroundProcessID: resp.ID,
Expand Down Expand Up @@ -316,7 +357,13 @@ func executeForegroundToolCall(
return toolCallStartErrorResult(id, "start process", err)
}

result := waitForToolCallProcess(ctx, conn, clock, resp.ID, timeout)
// After an interrupt's cancel the process has stopped, so the
// agent answers without waiting.
waitBound := agentOutputWaitCap + AgentAnswerTimeout
if id.Cause == ToolCallCauseInterrupt {
waitBound = AgentAnswerTimeout
}
result := waitForToolCallProcess(ctx, conn, clock, resp.ID, timeout, waitBound)
if note := detectFileDump(args.Command); note != "" {
result.Note = note
}
Expand All @@ -325,33 +372,25 @@ func executeForegroundToolCall(

// waitForToolCallProcess waits on the output of a process started with
// an execute deadline until the agent reports that it exited or timed
// out. A read the agent does not answer is sent again until the agent's
// wait cap plus AgentAnswerTimeout ends.
// out. A read the agent does not answer is sent again until waitBound
// ends: the agent's wait cap plus AgentAnswerTimeout in generation.
func waitForToolCallProcess(
ctx context.Context,
conn workspacesdk.AgentConn,
clock quartz.Clock,
processID string,
timeout time.Duration,
waitBound time.Duration,
) ExecuteResult {
for {
var resp workspacesdk.ProcessOutputResponse
err := requestUntilAnsweredWithin(ctx, clock, agentOutputWaitCap+AgentAnswerTimeout, func(ctx context.Context) error {
err := requestUntilAnsweredWithin(ctx, clock, waitBound, func(ctx context.Context) error {
var err error
resp, err = conn.ProcessOutput(ctx, processID, &workspacesdk.ProcessOutputOptions{Wait: true})
return err
})
if err != nil {
// The command may have exited or passed its deadline, so
// neither is claimed.
return ExecuteResult{
Success: false,
ExitCode: -1,
Error: UnknownOutcome(fmt.Sprintf("the command's result could not be read (%v)", err),
"the command may still be running or may have finished",
fmt.Sprintf("Check it with process_output using process ID %s.", processID)),
BackgroundProcessID: processID,
}
return unreadOutputResult(processID, err)
}
var result ExecuteResult
switch {
Expand All @@ -370,6 +409,57 @@ func waitForToolCallProcess(
}
}

// unreadOutputResult is the result when the output of processID could
// not be read. The command may have exited or passed its deadline, so
// neither is claimed.
func unreadOutputResult(processID string, err error) ExecuteResult {
return ExecuteResult{
Success: false,
ExitCode: -1,
Error: UnknownOutcome(fmt.Sprintf("the command's result could not be read (%v)", err),
"the command may still be running or may have finished",
fmt.Sprintf("Check it with process_output using process ID %s.", processID)),
BackgroundProcessID: processID,
}
}

// readCanceledBackgroundProcess reads the output of the background
// process processID once after an interrupt canceled its tool call,
// until the agent answers or AgentAnswerTimeout ends. A process that is
// still running, because it was not stopped, keeps the background start
// result.
func readCanceledBackgroundProcess(
ctx context.Context,
conn workspacesdk.AgentConn,
clock quartz.Clock,
processID string,
) ExecuteResult {
var resp workspacesdk.ProcessOutputResponse
err := RequestUntilAnswered(ctx, clock, func(ctx context.Context) error {
var err error
resp, err = conn.ProcessOutput(ctx, processID, nil)
return err
})
if err != nil {
return unreadOutputResult(processID, err)
}
var result ExecuteResult
switch {
case resp.Running:
return ExecuteResult{
Success: true,
BackgroundProcessID: processID,
Backgrounded: true,
}
case resp.Canceled:
result = canceledResult(resp)
default:
result = completedResult(resp)
}
result.WallDurationMs = resp.DurationMs
return result
}

// completedResult builds the result for an exited process, treating a
// missing exit code as success.
func completedResult(resp workspacesdk.ProcessOutputResponse) ExecuteResult {
Expand Down
Loading
Loading