Skip to content

Commit b28362f

Browse files
committed
feat: trigger compaction independently per chat and compaction model
Compaction fires at whichever of two independent triggers is reached first: the chat trigger (user or chat model threshold against the chat model's window) and the organization compaction override model's own trigger (its compression threshold against its own window). The enabled trigger with the lower token point binds and flows through the existing compaction plumbing. A threshold of 100 disables only its own trigger. The UI mirrors the backend rules from the existing member-readable model-overrides and models endpoints: the org settings compaction row warns when the selected model's trigger undercuts chat models' default trigger points, the user thresholds page shows the effective trigger in its own column with an organization-override popover, and the chat context gauge annotates the compact-at percent when the organization trigger binds.
1 parent e258526 commit b28362f

26 files changed

Lines changed: 1358 additions & 262 deletions

‎coderd/x/chatd/ARCHITECTURE.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -964,6 +964,8 @@ The model editor scopes the field to openai-typed providers with a `providers` s
964964

965965
#### Compaction model selection
966966

967+
TODO: Document independent chat and override compaction triggers and the shared post-compaction limit check.
968+
967969
Compaction is an auxiliary LLM call: when the conversation approaches the context limit, the generation goroutine asks a model to summarize the history, commits the summary as a compressed boundary, and continues the turn on the chat model.
968970

969971
By default the summary is generated with the chat model. Organization admins can select a dedicated compaction model via `PUT /api/v2/organizations/{organization}/chats/model-overrides/compaction`. The selection is stored as a typed `chat_organization_model_overrides` row and resolved using the chat's organization. Its composite foreign key binds the model config UUID to that organization, so cross-organization and malformed string references cannot be stored. The override affects only the summary call; compressed-message storage and the post-compaction assistant generation keep using the chat model.

‎coderd/x/chatd/chatd_test.go‎

Lines changed: 143 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -6365,7 +6365,7 @@ func TestActiveServer_CompactionModelOverride(t *testing.T) {
63656365
})
63666366
}
63676367

6368-
t.Run("compaction triggers at the stricter override context limit", func(t *testing.T) {
6368+
t.Run("compaction triggers at the override model's own threshold", func(t *testing.T) {
63696369
t.Parallel()
63706370

63716371
ctx := testutil.Context(t, testutil.WaitLong)
@@ -6392,10 +6392,6 @@ func TestActiveServer_CompactionModelOverride(t *testing.T) {
63926392
}
63936393
})
63946394
user, org, model := seedAnthropicChatDependencies(t, db, anthropicURL)
6395-
// The chat model alone would not compact: 80 tokens of usage is
6396-
// 8% of its 1000-token limit. The override model's 100-token
6397-
// limit makes the effective threshold 70 tokens, so compaction
6398-
// must trigger.
63996395
model = updateChatModelCompressionThreshold(t, db, model, 1_000, thresholdPercent)
64006396
seedOverrideModel(ctx, t, db, model, overrideModelName, "high", 100)
64016397
ws, dbAgent := seedWorkspaceWithAgent(t, db, user.ID)
@@ -6435,6 +6431,148 @@ func TestActiveServer_CompactionModelOverride(t *testing.T) {
64356431
require.Len(t, compressed.summaries, 1)
64366432
requireTextPart(t, messages[len(messages)-1], "continued after compaction")
64376433
})
6434+
6435+
t.Run("user threshold 100 does not disable the override model trigger", func(t *testing.T) {
6436+
t.Parallel()
6437+
6438+
ctx := testutil.Context(t, testutil.WaitLong)
6439+
db, ps := dbtestutil.NewDB(t)
6440+
var streamCount atomic.Int32
6441+
anthropicURL := chattest.NewAnthropic(t, func(req *chattest.AnthropicRequest) chattest.AnthropicResponse {
6442+
body := anthropicRequestBody(t, *req)
6443+
if !req.Stream {
6444+
if strings.Contains(body, "You are performing a context compaction") {
6445+
require.Equal(t, overrideModelName, req.Model)
6446+
return anthropicCompactionResponse(compactionSummary)
6447+
}
6448+
return chattest.AnthropicNonStreamingResponse("title")
6449+
}
6450+
switch streamCount.Add(1) {
6451+
case 1:
6452+
return highUsageReadFileResponse("/tmp/a.txt")
6453+
default:
6454+
require.Contains(t, body, compactionSummary)
6455+
return chattest.AnthropicStreamingResponse(chattest.AnthropicTextChunksWithCacheUsage(chattest.AnthropicUsage{
6456+
InputTokens: 20,
6457+
OutputTokens: 5,
6458+
}, "continued after compaction")...)
6459+
}
6460+
})
6461+
user, org, model := seedAnthropicChatDependencies(t, db, anthropicURL)
6462+
model = updateChatModelCompressionThreshold(t, db, model, 1_000, thresholdPercent)
6463+
_, err := db.UpdateUserChatCompactionThreshold(ctx, database.UpdateUserChatCompactionThresholdParams{
6464+
UserID: user.ID,
6465+
Key: codersdk.CompactionThresholdKey(model.ID),
6466+
ThresholdPercent: 100,
6467+
})
6468+
require.NoError(t, err)
6469+
seedOverrideModel(ctx, t, db, model, overrideModelName, "high", 100)
6470+
ws, dbAgent := seedWorkspaceWithAgent(t, db, user.ID)
6471+
6472+
ctrl := gomock.NewController(t)
6473+
mockConn := agentconnmock.NewMockAgentConn(ctrl)
6474+
setupToolExecutionAgentConn(t, mockConn)
6475+
mockConn.EXPECT().ReadFileLines(gomock.Any(), "/tmp/a.txt", int64(1), int64(0), gomock.Any()).
6476+
Return(workspacesdk.ReadFileLinesResponse{Success: true, FileSize: 12, TotalLines: 1, LinesRead: 1, Content: "1\tpackage main"}, nil).
6477+
Times(1)
6478+
6479+
server := newActiveTestServer(t, db, ps, func(cfg *chatd.Config) {
6480+
cfg.AIBridgeTransportFactory = chatAIGatewayTransportFactoryPointer(chattest.NewMockAIBridgeTransport(t, anthropicURL, chattest.WithPreservePath()))
6481+
cfg.AgentConn = func(_ context.Context, agentID uuid.UUID) (workspacesdk.AgentConn, func(), error) {
6482+
require.Equal(t, dbAgent.ID, agentID)
6483+
return mockConn, func() {}, nil
6484+
}
6485+
})
6486+
chat, err := server.CreateChat(ctx, chatd.CreateOptions{
6487+
OrganizationID: org.ID,
6488+
OwnerID: user.ID,
6489+
WorkspaceID: uuid.NullUUID{UUID: ws.ID, Valid: true},
6490+
AgentID: uuid.NullUUID{UUID: dbAgent.ID, Valid: true},
6491+
Title: "compaction-user-disabled",
6492+
ModelConfigID: model.ID,
6493+
InitialUserContent: []codersdk.ChatMessagePart{
6494+
codersdk.ChatMessageText("read the file and continue"),
6495+
},
6496+
})
6497+
require.NoError(t, err)
6498+
waitForChatStatus(ctx, t, db, chat.ID, database.ChatStatusWaiting)
6499+
6500+
messages := chatMessages(ctx, t, db, chat.ID)
6501+
promptMessages, err := db.GetChatMessagesForPromptByChatID(ctx, chat.ID)
6502+
require.NoError(t, err)
6503+
compressed := compressedChatSummarizedMessages(t, append(promptMessages, messages...))
6504+
require.Len(t, compressed.summaries, 1)
6505+
requireTextPart(t, messages[len(messages)-1], "continued after compaction")
6506+
})
6507+
6508+
t.Run("override model with a higher own point does not compact early", func(t *testing.T) {
6509+
t.Parallel()
6510+
6511+
ctx := testutil.Context(t, testutil.WaitLong)
6512+
db, ps := dbtestutil.NewDB(t)
6513+
var streamCount atomic.Int32
6514+
var compactionRequests atomic.Int32
6515+
anthropicURL := chattest.NewAnthropic(t, func(req *chattest.AnthropicRequest) chattest.AnthropicResponse {
6516+
body := anthropicRequestBody(t, *req)
6517+
if !req.Stream {
6518+
if strings.Contains(body, "You are performing a context compaction") {
6519+
compactionRequests.Add(1)
6520+
return anthropicCompactionResponse(compactionSummary)
6521+
}
6522+
return chattest.AnthropicNonStreamingResponse("title")
6523+
}
6524+
switch streamCount.Add(1) {
6525+
case 1:
6526+
return highUsageReadFileResponse("/tmp/a.txt")
6527+
default:
6528+
return chattest.AnthropicStreamingResponse(chattest.AnthropicTextChunksWithCacheUsage(chattest.AnthropicUsage{
6529+
InputTokens: 20,
6530+
OutputTokens: 5,
6531+
}, "continued without compaction")...)
6532+
}
6533+
})
6534+
user, org, model := seedAnthropicChatDependencies(t, db, anthropicURL)
6535+
model = updateChatModelCompressionThreshold(t, db, model, 1_000, thresholdPercent)
6536+
overrideModel := seedOverrideModel(ctx, t, db, model, overrideModelName, "high", 100)
6537+
updateChatModelCompressionThreshold(t, db, overrideModel, 100, 95)
6538+
ws, dbAgent := seedWorkspaceWithAgent(t, db, user.ID)
6539+
6540+
ctrl := gomock.NewController(t)
6541+
mockConn := agentconnmock.NewMockAgentConn(ctrl)
6542+
setupToolExecutionAgentConn(t, mockConn)
6543+
mockConn.EXPECT().ReadFileLines(gomock.Any(), "/tmp/a.txt", int64(1), int64(0), gomock.Any()).
6544+
Return(workspacesdk.ReadFileLinesResponse{Success: true, FileSize: 12, TotalLines: 1, LinesRead: 1, Content: "1\tpackage main"}, nil).
6545+
Times(1)
6546+
6547+
server := newActiveTestServer(t, db, ps, func(cfg *chatd.Config) {
6548+
cfg.AIBridgeTransportFactory = chatAIGatewayTransportFactoryPointer(chattest.NewMockAIBridgeTransport(t, anthropicURL, chattest.WithPreservePath()))
6549+
cfg.AgentConn = func(_ context.Context, agentID uuid.UUID) (workspacesdk.AgentConn, func(), error) {
6550+
require.Equal(t, dbAgent.ID, agentID)
6551+
return mockConn, func() {}, nil
6552+
}
6553+
})
6554+
chat, err := server.CreateChat(ctx, chatd.CreateOptions{
6555+
OrganizationID: org.ID,
6556+
OwnerID: user.ID,
6557+
WorkspaceID: uuid.NullUUID{UUID: ws.ID, Valid: true},
6558+
AgentID: uuid.NullUUID{UUID: dbAgent.ID, Valid: true},
6559+
Title: "compaction-override-higher-point",
6560+
ModelConfigID: model.ID,
6561+
InitialUserContent: []codersdk.ChatMessagePart{
6562+
codersdk.ChatMessageText("read the file and continue"),
6563+
},
6564+
})
6565+
require.NoError(t, err)
6566+
waitForChatStatus(ctx, t, db, chat.ID, database.ChatStatusWaiting)
6567+
6568+
messages := chatMessages(ctx, t, db, chat.ID)
6569+
promptMessages, err := db.GetChatMessagesForPromptByChatID(ctx, chat.ID)
6570+
require.NoError(t, err)
6571+
compressed := compressedChatSummarizedMessages(t, append(promptMessages, messages...))
6572+
require.Empty(t, compressed.summaries)
6573+
require.Zero(t, compactionRequests.Load())
6574+
requireTextPart(t, messages[len(messages)-1], "continued without compaction")
6575+
})
64386576
}
64396577

64406578
func TestActiveServer_AnthropicModelReasoningEffort(t *testing.T) {

‎coderd/x/chatd/compaction_override.go‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -47,11 +47,9 @@ type resolvedCompactionOverride struct {
4747
ResolvedModel string
4848
}
4949

50-
// resolveCompactionOverrideConfig resolves the stored organization-scoped
51-
// compaction model override. Unset, stale, and credential-less
52-
// overrides fall back to the chat model (nil override). This runs on every
53-
// generation prepare because the override's context limit feeds the
54-
// compaction trigger; the model client is built only when compaction runs.
50+
// resolveCompactionOverrideConfig treats unset, stale, or credential-less
51+
// overrides as absent. Preparation needs their trigger settings, but model
52+
// client construction is deferred until compaction runs.
5553
func (p *Server) resolveCompactionOverrideConfig(
5654
ctx context.Context,
5755
chat database.Chat,

‎coderd/x/chatd/generation.go‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -276,11 +276,9 @@ func generationCompactionThreshold(compaction *generationCompaction) int32 {
276276
return compaction.Options.ThresholdPercent
277277
}
278278

279-
// generationCompactionContextLimit returns the context limit the compaction
280-
// trigger was evaluated against at prepare time (the stricter of the chat and
281-
// override models' limits). The still-over-limit check must compare against
282-
// the same limit, otherwise a stricter override loops through repeated
283-
// compactions instead of surfacing errCompactionStillOverLimit.
279+
// generationCompactionContextLimit returns the limit used to select the
280+
// compaction trigger. The still-over-limit check must use the same limit to
281+
// avoid repeated compactions instead of surfacing errCompactionStillOverLimit.
284282
func generationCompactionContextLimit(compaction *generationCompaction) int64 {
285283
if compaction == nil {
286284
return 0

‎coderd/x/chatd/generation_preparer.go‎

Lines changed: 43 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -713,31 +713,34 @@ func (server *Server) prepareGeneration(
713713
if override, ok := server.resolveUserCompactionThreshold(ctx, chat.OwnerID, modelConfig.ID); ok {
714714
effectiveThreshold = override
715715
}
716-
// The compaction trigger uses the stricter of the chat and override
717-
// models' context limits: the history must also fit the summarizer's
718-
// window.
719-
compactionContextLimit := modelConfig.ContextLimit
716+
chatTrigger := compactionTrigger{
717+
thresholdPercent: effectiveThreshold,
718+
contextLimit: modelConfig.ContextLimit,
719+
}
720720
compactionOverride, err := server.resolveCompactionOverrideConfig(ctx, chat)
721721
if err != nil {
722722
cleanup()
723723
return generationPrepared{}, err
724724
}
725+
// Each threshold applies to its model's window. The earliest enabled
726+
// trigger binds so history also fits the compaction model.
727+
binding := chatTrigger
725728
if compactionOverride != nil {
726-
if overrideLimit := compactionOverride.Config.ContextLimit; overrideLimit > 0 &&
727-
(compactionContextLimit <= 0 || overrideLimit < compactionContextLimit) {
728-
compactionContextLimit = overrideLimit
729-
}
729+
binding = bindingCompactionTrigger(chatTrigger, compactionTrigger{
730+
thresholdPercent: compactionOverride.Config.CompressionThreshold,
731+
contextLimit: compactionOverride.Config.ContextLimit,
732+
})
730733
}
731734
compactionStepUsage := latestPromptUsage(promptRows)
732-
compactionNeeded := shouldCompactPromptUsage(compactionStepUsage, compactionContextLimit, effectiveThreshold)
735+
compactionNeeded := shouldCompactPromptUsage(compactionStepUsage, binding.contextLimit, binding.thresholdPercent)
733736
// The options carry the chat model; generateCompaction swaps in the
734737
// override client when one is configured.
735738
compactionOptions := chatloop.GenerateCompactionOptions{
736739
Model: model.LanguageModel(),
737740
Messages: compactionPromptMessages,
738-
ThresholdPercent: effectiveThreshold,
739-
ContextLimit: compactionContextLimit,
740-
ContextLimitFallback: compactionContextLimit,
741+
ThresholdPercent: binding.thresholdPercent,
742+
ContextLimit: binding.contextLimit,
743+
ContextLimitFallback: binding.contextLimit,
741744
ToolCallID: compactionToolCallID,
742745
ToolName: "chat_summarized",
743746
DebugSvc: debugSvc,
@@ -801,6 +804,34 @@ func latestPromptUsage(messages []database.ChatMessage) fantasy.Usage {
801804
return fantasy.Usage{}
802805
}
803806

807+
type compactionTrigger struct {
808+
thresholdPercent int32
809+
contextLimit int64
810+
}
811+
812+
func (t compactionTrigger) enabled() bool {
813+
return t.thresholdPercent >= 0 && t.thresholdPercent < 100 && t.contextLimit > 0
814+
}
815+
816+
func (t compactionTrigger) point() float64 {
817+
return float64(t.contextLimit) * float64(t.thresholdPercent) / 100
818+
}
819+
820+
// When both triggers are enabled, the lower token point binds because they
821+
// evaluate the same prompt usage.
822+
func bindingCompactionTrigger(chat, override compactionTrigger) compactionTrigger {
823+
switch {
824+
case !override.enabled():
825+
return chat
826+
case !chat.enabled():
827+
return override
828+
case override.point() < chat.point():
829+
return override
830+
default:
831+
return chat
832+
}
833+
}
834+
804835
func shouldCompactPromptUsage(usage fantasy.Usage, contextLimit int64, thresholdPercent int32) bool {
805836
if thresholdPercent >= 100 || contextLimit <= 0 {
806837
return false

‎coderd/x/chatd/generation_preparer_internal_test.go‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -643,6 +643,82 @@ func TestShouldCompactPromptUsage(t *testing.T) {
643643
})
644644
}
645645

646+
func TestBindingCompactionTrigger(t *testing.T) {
647+
t.Parallel()
648+
649+
chat := func(threshold int32, limit int64) compactionTrigger {
650+
return compactionTrigger{thresholdPercent: threshold, contextLimit: limit}
651+
}
652+
653+
cases := []struct {
654+
name string
655+
chat compactionTrigger
656+
override compactionTrigger
657+
want compactionTrigger
658+
}{
659+
{
660+
name: "lower override point wins",
661+
chat: chat(70, 200_000),
662+
override: chat(70, 32_000),
663+
want: chat(70, 32_000),
664+
},
665+
{
666+
name: "lower chat point wins over higher override point",
667+
chat: chat(70, 100_000),
668+
override: chat(95, 80_000),
669+
want: chat(70, 100_000),
670+
},
671+
{
672+
name: "tie prefers the chat trigger",
673+
chat: chat(70, 100_000),
674+
override: chat(70, 100_000),
675+
want: chat(70, 100_000),
676+
},
677+
{
678+
name: "chat trigger disabled by threshold 100 yields override",
679+
chat: chat(100, 200_000),
680+
override: chat(70, 32_000),
681+
want: chat(70, 32_000),
682+
},
683+
{
684+
name: "chat trigger disabled by zero limit yields override",
685+
chat: chat(70, 0),
686+
override: chat(70, 32_000),
687+
want: chat(70, 32_000),
688+
},
689+
{
690+
name: "override disabled by threshold 100 yields chat",
691+
chat: chat(70, 200_000),
692+
override: chat(100, 32_000),
693+
want: chat(70, 200_000),
694+
},
695+
{
696+
name: "override disabled by zero limit yields chat",
697+
chat: chat(70, 200_000),
698+
override: chat(70, 0),
699+
want: chat(70, 200_000),
700+
},
701+
{
702+
name: "both disabled passes the chat pair through",
703+
chat: chat(100, 200_000),
704+
override: chat(100, 32_000),
705+
want: chat(100, 200_000),
706+
},
707+
{
708+
name: "threshold zero fires immediately and wins",
709+
chat: chat(70, 200_000),
710+
override: chat(0, 32_000),
711+
want: chat(0, 32_000),
712+
},
713+
}
714+
for _, tc := range cases {
715+
t.Run(tc.name, func(t *testing.T) {
716+
t.Parallel()
717+
assert.Equal(t, tc.want, bindingCompactionTrigger(tc.chat, tc.override))
718+
})
719+
}
720+
}
721+
646722
func TestEnabledMCPServerConfigsForChatOrg(t *testing.T) {
647723
t.Parallel()
648724

‎coderd/x/chatd/message_conversion.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -597,6 +597,11 @@ func hasUncompressedMessageAfter(messages []database.ChatMessage, index int) boo
597597
}
598598

599599
func postCompactionAssistantOverLimit(msg database.ChatMessage, thresholdPercent int32, contextLimit int64) bool {
600+
// Zero means "always compact." Excluding it from this check prevents a
601+
// terminal over-limit error after every compaction.
602+
if thresholdPercent <= 0 {
603+
return false
604+
}
600605
return shouldCompactPromptUsage(usageFromMessage(msg), contextLimit, thresholdPercent)
601606
}
602607

0 commit comments

Comments
 (0)