Skip to content
Merged
82 changes: 82 additions & 0 deletions coderd/x/chatd/active_turn_debug.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"cdr.dev/slog/v3"
"github.com/coder/coder/v2/coderd/database"
"github.com/coder/coder/v2/coderd/x/chatd/chatdebug"
"github.com/coder/coder/v2/coderd/x/chatd/mcpclient"
)

type runnerDebugTurn struct {
Expand Down Expand Up @@ -49,7 +50,14 @@ func (d *runnerDebugTurn) Ensure(

d.mu.Lock()
defer d.mu.Unlock()
return d.ensureLocked(ctx, chat, debug)
}

func (d *runnerDebugTurn) ensureLocked(
ctx context.Context,
chat database.Chat,
debug *generationDebug,
) context.Context {
// Check finalized/disabled before created: once the turn is
// finalized, new contexts must not be attributed to the
// finalized run, even if it was created earlier.
Expand All @@ -68,6 +76,17 @@ func (d *runnerDebugTurn) Ensure(
seedSummary := chatdebug.SeedSummary(
chatdebug.TruncateLabel(debug.TriggerLabel, chatdebug.MaxLabelLength),
)
// Carry per-server MCP connect outcomes (and their dropped
// count) stashed by RecordMCPConnectSummaries before the run
// existed, so slow or failing servers appear in the run instead
// of as a silent gap before the first step. Seeded keys survive
// FinalizeRun's summary aggregation.
for key, stashed := range d.seedSummary {
if seedSummary == nil {
seedSummary = make(map[string]any, len(d.seedSummary))
}
seedSummary[key] = stashed
}
rootChatID := uuid.Nil
if chat.RootChatID.Valid {
rootChatID = chat.RootChatID.UUID
Expand Down Expand Up @@ -121,6 +140,69 @@ func (d *runnerDebugTurn) Context(ctx context.Context) context.Context {
return d.contextLocked(ctx)
}

// RecordMCPConnectSummaries merges one preparation's per-server MCP
// connect outcomes into the mcp_connect summary key, creating the
// debug run if it does not exist yet. Preparation invokes it as soon
// as its MCP connect phase completes, so every attempt is recorded:
// preparations that fail after connecting, decision errors, and
// actions that never reach Ensure (local tool execution,
// requires-action, turn finishing). Creating the run here matters
// when the first preparation connects and then fails: no model or
// compaction action ever runs Ensure, and Finalize discards the
// stash of a never-created run.
func (d *runnerDebugTurn) RecordMCPConnectSummaries(
ctx context.Context,
chat database.Chat,
debug *generationDebug,
summaries []mcpclient.ConnectSummary,
) {
if d == nil || len(summaries) == 0 {
return
}
d.mu.Lock()
defer d.mu.Unlock()
if d.disabled || d.finalized {
return
}
// Merge before ensuring so the outcomes ride the seed summary
// when this call is the one that creates the run.
d.mergeMCPConnectSummariesLocked(summaries)
d.ensureLocked(ctx, chat, debug)
}

// maxMCPConnectSummaryEntries bounds the retained per-preparation MCP
// connect outcomes. A turn may run up to 1,200 generation steps, each
// reconnecting to every selected server, so unbounded retention could
// grow one run summary to megabytes of database and API payload. The
// newest entries win because the tail of the history is what shows a
// server that degraded mid-turn; the mcp_connect_dropped count keeps
// the truncation visible.
const maxMCPConnectSummaryEntries = 100

// mergeMCPConnectSummariesLocked appends a preparation's per-server
// MCP connect outcomes to the mcp_connect summary key. chatd
// reconnects to every configured MCP server on each generation step
// while the run is created only once, so without this merge only
// one preparation's outcomes would survive to the finalized run and
// a server that degrades mid-turn would still be reported as
// connected.
func (d *runnerDebugTurn) mergeMCPConnectSummariesLocked(summaries []mcpclient.ConnectSummary) {
if len(summaries) == 0 {
return
}
if d.seedSummary == nil {
d.seedSummary = make(map[string]any, 1)
}
existing, _ := d.seedSummary["mcp_connect"].([]mcpclient.ConnectSummary)
existing = append(existing, summaries...)
if over := len(existing) - maxMCPConnectSummaryEntries; over > 0 {
existing = existing[over:]
dropped, _ := d.seedSummary["mcp_connect_dropped"].(int)
d.seedSummary["mcp_connect_dropped"] = dropped + over
}
d.seedSummary["mcp_connect"] = existing
}

func (d *runnerDebugTurn) contextLocked(ctx context.Context) context.Context {
if !d.created || d.runContext.RunID == uuid.Nil {
return ctx
Expand Down
179 changes: 179 additions & 0 deletions coderd/x/chatd/active_turn_debug_internal_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ package chatd
import (
"context"
"database/sql"
"encoding/json"
"fmt"
"testing"

"github.com/google/uuid"
Expand All @@ -12,6 +14,7 @@ import (
"github.com/coder/coder/v2/coderd/database"
"github.com/coder/coder/v2/coderd/database/dbmock"
"github.com/coder/coder/v2/coderd/x/chatd/chatdebug"
"github.com/coder/coder/v2/coderd/x/chatd/mcpclient"
"github.com/coder/coder/v2/testutil"
)

Expand Down Expand Up @@ -153,3 +156,179 @@ func TestRunnerDebugTurnFinalizeOnce(t *testing.T) {
turn.Finalize(ctx)
turn.Finalize(ctx)
}

func TestRunnerDebugTurnRecordMCPConnectSummaries(t *testing.T) {
t.Parallel()

ctx := testutil.Context(t, testutil.WaitLong)
runnerCtx, cancel := context.WithCancel(ctx)
defer cancel()
ctrl := gomock.NewController(t)
db := dbmock.NewMockStore(ctrl)
chatID := uuid.New()
runID := uuid.New()
configID := uuid.New()
svc := chatdebug.NewService(db, testutil.Logger(t), nil)
turn := newRunnerDebugTurn(runnerCtx, testutil.Logger(t))

var seededSummary []byte
db.EXPECT().InsertChatDebugRun(gomock.Any(), gomock.Any()).
DoAndReturn(func(_ context.Context, params database.InsertChatDebugRunParams) (database.ChatDebugRun, error) {
require.True(t, params.Summary.Valid)
seededSummary = params.Summary.RawMessage
return database.ChatDebugRun{
ID: runID,
ChatID: chatID,
Kind: string(chatdebug.KindChatTurn),
Status: string(chatdebug.StatusInProgress),
}, nil
}).
Times(1)
db.EXPECT().GetChatDebugStepsByRunID(gomock.Any(), runID).Return(nil, nil).Times(1)
var finalSummary []byte
db.EXPECT().UpdateChatDebugRun(gomock.Any(), gomock.Any()).
DoAndReturn(func(_ context.Context, params database.UpdateChatDebugRunParams) (database.ChatDebugRun, error) {
require.True(t, params.Summary.Valid)
finalSummary = params.Summary.RawMessage
return database.ChatDebugRun{ID: runID, ChatID: chatID}, nil
}).Times(1)

debug := generationDebug{
Enabled: true,
Service: svc,
TriggerMessageID: 1,
ModelConfig: database.ChatModelConfig{ID: uuid.New()},
}
first := []mcpclient.ConnectSummary{{
ConfigID: configID,
Slug: "registry",
Outcome: mcpclient.ConnectOutcomeConnected,
DurationMS: 17,
ToolCount: 1,
}}
second := []mcpclient.ConnectSummary{{
ConfigID: configID,
Slug: "registry",
Outcome: mcpclient.ConnectOutcomeTimeout,
DurationMS: 10000,
Error: "connect: context deadline exceeded",
}}

// Preparation records each attempt as soon as its connect phase
// completes; the first record creates the run with the outcome
// already seeded, and a later Ensure must not create a second
// run.
turn.RecordMCPConnectSummaries(ctx, database.Chat{ID: chatID}, &debug, first)
turn.Ensure(ctx, database.Chat{ID: chatID}, &debug)
// A later generation step reconnects and reports a degraded
// outcome for the same server; its action may never reach
// Ensure, and the outcome must still survive to the finalized
// summary.
turn.RecordMCPConnectSummaries(ctx, database.Chat{ID: chatID}, &debug, second)
turn.RecordOutcome(chatdebug.StatusCompleted)
turn.Finalize(ctx)

var seeded struct {
MCPConnect []mcpclient.ConnectSummary `json:"mcp_connect"`
}
require.NoError(t, json.Unmarshal(seededSummary, &seeded))
require.Len(t, seeded.MCPConnect, 1)
require.Equal(t, mcpclient.ConnectOutcomeConnected, seeded.MCPConnect[0].Outcome)

var summary struct {
MCPConnect []mcpclient.ConnectSummary `json:"mcp_connect"`
}
require.NoError(t, json.Unmarshal(finalSummary, &summary))
require.Len(t, summary.MCPConnect, 2)
require.Equal(t, mcpclient.ConnectOutcomeConnected, summary.MCPConnect[0].Outcome)
require.Equal(t, mcpclient.ConnectOutcomeTimeout, summary.MCPConnect[1].Outcome)
}

func TestRunnerDebugTurnBoundsMCPConnectSummaries(t *testing.T) {
t.Parallel()

ctx := testutil.Context(t, testutil.WaitLong)
runnerCtx, cancel := context.WithCancel(ctx)
defer cancel()
ctrl := gomock.NewController(t)
db := dbmock.NewMockStore(ctrl)
chatID := uuid.New()
runID := uuid.New()
svc := chatdebug.NewService(db, testutil.Logger(t), nil)
turn := newRunnerDebugTurn(runnerCtx, testutil.Logger(t))

var seededSummary []byte
db.EXPECT().InsertChatDebugRun(gomock.Any(), gomock.Any()).
DoAndReturn(func(_ context.Context, params database.InsertChatDebugRunParams) (database.ChatDebugRun, error) {
require.True(t, params.Summary.Valid)
seededSummary = params.Summary.RawMessage
return database.ChatDebugRun{
ID: runID,
ChatID: chatID,
Kind: string(chatdebug.KindChatTurn),
Status: string(chatdebug.StatusInProgress),
}, nil
}).
Times(1)
db.EXPECT().GetChatDebugStepsByRunID(gomock.Any(), runID).Return(nil, nil).Times(1)
var finalSummary []byte
db.EXPECT().UpdateChatDebugRun(gomock.Any(), gomock.Any()).
DoAndReturn(func(_ context.Context, params database.UpdateChatDebugRunParams) (database.ChatDebugRun, error) {
require.True(t, params.Summary.Valid)
finalSummary = params.Summary.RawMessage
return database.ChatDebugRun{ID: runID, ChatID: chatID}, nil
}).Times(1)

debug := generationDebug{
Enabled: true,
Service: svc,
TriggerMessageID: 1,
ModelConfig: database.ChatModelConfig{ID: uuid.New()},
}
// 25 preparations against 5 servers produce 125 outcomes,
// overflowing the cap. The first record creates the run.
for prep := 0; prep < 25; prep++ {
batch := make([]mcpclient.ConnectSummary, 5)
for server := range batch {
batch[server] = mcpclient.ConnectSummary{
ConfigID: uuid.New(),
Slug: fmt.Sprintf("server-%d", server),
Outcome: mcpclient.ConnectOutcomeConnected,
DurationMS: int64(prep),
}
}
turn.RecordMCPConnectSummaries(ctx, database.Chat{ID: chatID}, &debug, batch)
}

type boundedSummary struct {
MCPConnect []mcpclient.ConnectSummary `json:"mcp_connect"`
MCPConnectDropped int `json:"mcp_connect_dropped"`
}
// The run was created by the first record, so the seed carries
// only that preparation's outcomes.
var seeded boundedSummary
require.NoError(t, json.Unmarshal(seededSummary, &seeded))
require.Len(t, seeded.MCPConnect, 5)
require.Zero(t, seeded.MCPConnectDropped)

// One more preparation still respects the cap and grows the
// dropped count.
turn.RecordMCPConnectSummaries(ctx, database.Chat{ID: chatID}, &debug, []mcpclient.ConnectSummary{{
ConfigID: uuid.New(),
Slug: "server-0",
Outcome: mcpclient.ConnectOutcomeTimeout,
DurationMS: 10000,
}})
turn.RecordOutcome(chatdebug.StatusCompleted)
turn.Finalize(ctx)

var final boundedSummary
require.NoError(t, json.Unmarshal(finalSummary, &final))
require.Len(t, final.MCPConnect, maxMCPConnectSummaryEntries)
require.Equal(t, 26, final.MCPConnectDropped)
// The newest outcomes win: the oldest five preparations were
// dropped, so the retained history starts at preparation 5 and
// ends with the timeout recorded above.
require.Equal(t, int64(5), final.MCPConnect[0].DurationMS)
require.Equal(t, mcpclient.ConnectOutcomeTimeout, final.MCPConnect[maxMCPConnectSummaryEntries-1].Outcome)
}
6 changes: 6 additions & 0 deletions coderd/x/chatd/chatd.go
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,12 @@ const (
DefaultChatHeartbeatInterval = 30 * time.Second
maxChatSteps = 1200

// slowPrepareThreshold is the generation-preparation duration
// above which a warning is logged. Preparation runs before
// every generation step, so sustained slowness (workspace
// dials, MCP connects) taxes the whole turn.
slowPrepareThreshold = 30 * time.Second

// maxConcurrentRecordingUploads caps the number of recording
// stop-and-store operations that can run concurrently. Each
// slot buffers up to MaxRecordingSize + MaxThumbnailSize
Expand Down
Loading
Loading