Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
beda96e
feat: evaluate experiment rules per user in discovery and chatd
ThomasK33 Sep 25, 2026
aa3081d
test: satisfy lint in experiment rule enforcement tests
ThomasK33 Sep 25, 2026
275adf2
test: tidy experiment rule enforcement test helpers
ThomasK33 Sep 25, 2026
eb21733
test(site/src/modules/dashboard): cover a failed experiments refetch …
ThomasK33 Sep 25, 2026
4925010
fix(coderd/x/chatd): decide mcp-tool-search once per turn
ThomasK33 Sep 26, 2026
833a537
test(site/src/modules/dashboard): assert provider state after a faile…
ThomasK33 Sep 26, 2026
85b3853
test(coderd/x/chatd): pass turn experiment decisions to prepareGenera…
ThomasK33 Sep 26, 2026
5622217
fix(coderd/x/chatd): do not cache the turn decision without a prompt row
ThomasK33 Sep 26, 2026
421944a
fix(coderd/x/chatd): keep the newer turn's experiment decision
ThomasK33 Sep 26, 2026
8acc481
docs(coderd/x/chatd): leave an architecture TODO for the per-turn exp…
ThomasK33 Sep 26, 2026
1f14f3a
refactor(site/src/api/queries): inline experiments stale time and use…
ThomasK33 Sep 28, 2026
1a1da2a
test(site/src/modules/dashboard): drop comments that restate the refe…
ThomasK33 Sep 28, 2026
2baf8d2
docs: update ARCHITECTURE.md
ThomasK33 Sep 28, 2026
4fad71d
test(site/src): use React.FC in the experiments and dashboard tests
ThomasK33 Sep 28, 2026
cf0e7ad
fix(coderd/x/chatd): skip the mcp-tool-search rule read without MCP c…
ThomasK33 Sep 29, 2026
0a167da
docs(coderd/x/chatd): note that turns without MCP candidates skip the…
ThomasK33 Sep 29, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 21 additions & 3 deletions coderd/coderd.go
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ import (
"github.com/coder/coder/v2/coderd/database/dbtime"
"github.com/coder/coder/v2/coderd/database/pubsub"
"github.com/coder/coder/v2/coderd/entitlements"
experimentrules "github.com/coder/coder/v2/coderd/experiments"
"github.com/coder/coder/v2/coderd/externalauth"
"github.com/coder/coder/v2/coderd/files"
"github.com/coder/coder/v2/coderd/gitsshkey"
Expand Down Expand Up @@ -729,6 +730,15 @@ func New(options *Options) *API {
AISeatTracker: aiseats.Noop{},
}

api.ExperimentEvaluator, err = experimentrules.New(
options.Logger.Named("experiments"),
experimentrules.NewDBStore(options.Database),
experiments,
)
if err != nil {
panic(xerrors.Errorf("create experiment evaluator: %w", err))
}

api.WorkspaceAppsProvider = workspaceapps.NewDBTokenProvider(
ctx,
options.Logger.Named("workspaceapps"),
Expand Down Expand Up @@ -775,11 +785,11 @@ func New(options *Options) *API {
Logger: options.Logger.Named("site"),
AIGatewayEnabled: options.DeploymentValues.AI.BridgeConfig.Enabled.Value(),
UserSecretFilePathEnabled: !options.DeploymentValues.DisableUserSecretFilePath.Value(),
ExperimentEvaluator: api.ExperimentEvaluator,
})
if err != nil {
options.Logger.Fatal(ctx, "failed to initialize site handler", slog.Error(err))
}
api.SiteHandler.Experiments.Store(&experiments)

if options.UpdateCheckOptions != nil {
api.updateChecker = updatecheck.New(
Expand Down Expand Up @@ -926,7 +936,7 @@ func New(options *Options) *API {
options.PrometheusRegistry,
)
}
api.chatDaemon = chatd.New(options.Pubsub, chatd.Config{
api.chatDaemon, err = chatd.New(options.Pubsub, chatd.Config{
Logger: options.Logger.Named("chatd"),
Database: options.Database,
ReplicaID: api.ID,
Expand All @@ -940,6 +950,7 @@ func New(options *Options) *API {
StreamSilenceTimeout: streamSilenceTimeout,
DisableCallerSuppliedTools: options.DeploymentValues.DisableChatCallerSuppliedTools.Value(),
Experiments: experiments,
ExperimentEvaluator: api.ExperimentEvaluator,
AgentConn: api.agentProvider.AgentConn,
AgentInactiveDisconnectTimeout: api.AgentInactiveDisconnectTimeout,
CreateWorkspace: api.chatCreateWorkspace,
Expand All @@ -956,6 +967,9 @@ func New(options *Options) *API {
Auditor: &api.Auditor,
Limits: api.chatLimits,
})
if err != nil {
panic(xerrors.Errorf("create chat daemon: %w", err))
}
if !options.ChatWorkerDisabled {
api.chatDaemon.Start()
}
Expand Down Expand Up @@ -1414,7 +1428,7 @@ func New(options *Options) *API {
r.Route("/experiments", func(r chi.Router) {
r.Use(apiKeyMiddleware)
r.Get("/available", handleExperimentsAvailable)
r.Get("/", api.handleExperimentsGet)
r.With(httpmw.NoStore).Get("/", api.handleExperimentsGet)
})
api.registerChatAPIRoutes(r, apiKeyMiddleware)
r.Route("/mcp", func(r chi.Router) {
Expand Down Expand Up @@ -2247,6 +2261,10 @@ type API struct {
// Experiments contains the list of experiments currently enabled.
// This is used to gate features that are not yet ready for production.
Experiments codersdk.Experiments
// ExperimentEvaluator decides user-scoped experiments per user from
// stored rules, falling back to Experiments. Use it instead of
// Experiments for every experiment in codersdk.ExperimentsUserScoped.
ExperimentEvaluator *experimentrules.Evaluator

healthCheckGroup *singleflight.Group[string, *healthsdk.HealthcheckReport]
healthCheckCache atomic.Pointer[healthsdk.HealthcheckReport]
Expand Down
6 changes: 5 additions & 1 deletion coderd/experiments.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"net/http"

"github.com/coder/coder/v2/coderd/httpapi"
"github.com/coder/coder/v2/coderd/httpmw"
"github.com/coder/coder/v2/codersdk"
)

Expand All @@ -16,7 +17,10 @@ import (
// @Router /api/v2/experiments [get]
func (api *API) handleExperimentsGet(rw http.ResponseWriter, r *http.Request) {
ctx := r.Context()
httpapi.Write(ctx, rw, http.StatusOK, api.Experiments)
apiKey := httpmw.APIKey(r)
// The result is personalized by the caller's rules, so it is served
// with no-store caching (see the route registration).
httpapi.Write(ctx, rw, http.StatusOK, api.ExperimentEvaluator.EnabledExperiments(ctx, apiKey.UserID))
Comment on lines +21 to +23

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not totally in love with this endpoint now only returning the user's experiments tbh, but im not sure what the alternative is...

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yep, I agree. Personally, I’d prefer folding it into the stable experiments endpoint. For now, I don’t want to introduce experimental changes to a stable API.

}

// @Summary Get safe experiments
Expand Down
54 changes: 54 additions & 0 deletions coderd/experiments_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,15 @@ package coderd_test

import (
"context"
"fmt"
"net/http"
"testing"

"github.com/stretchr/testify/require"

"github.com/coder/coder/v2/coderd/coderdtest"
"github.com/coder/coder/v2/coderd/database/dbauthz"
experimentrules "github.com/coder/coder/v2/coderd/experiments"
"github.com/coder/coder/v2/coderd/httpmw"
"github.com/coder/coder/v2/codersdk"
"github.com/coder/coder/v2/testutil"
Expand Down Expand Up @@ -117,6 +121,56 @@ func Test_Experiments(t *testing.T) {
require.ErrorContains(t, err, httpmw.SignedOutErrorMessage)
})

t.Run("rules per user", func(t *testing.T) {
t.Parallel()
cfg := coderdtest.DeploymentValues(t)
cfg.Experiments = []string{"foo", string(codersdk.ExperimentMCPToolSearch)}
ownerClient, db := coderdtest.NewWithDatabase(t, &coderdtest.Options{
DeploymentValues: cfg,
})
owner := coderdtest.CreateFirstUser(t, ownerClient)
memberClient, member := coderdtest.CreateAnotherUser(t, ownerClient, owner.OrganizationID)

ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
defer cancel()

revisions := map[codersdk.Experiment]int64{}
writeRule := func(ex codersdk.Experiment, rule experimentrules.Rule) {
t.Helper()
_, stored, _, err := experimentrules.WriteRule(dbauthz.AsSystemRestricted(ctx), db, owner.UserID, ex, rule, revisions[ex])
require.NoError(t, err)
revisions[ex] = stored.Revision
}
requireExperiments := func(client *codersdk.Client, want ...codersdk.Experiment) {
t.Helper()
got, err := client.Experiments(ctx)
require.NoError(t, err)
require.ElementsMatch(t, want, got)
}

// A condition enables a user-scoped experiment only for matching
// users; a kill switch disables a statically enabled one.
writeRule(codersdk.ExperimentExample, experimentrules.Rule{
Mode: experimentrules.ModeCondition,
Condition: fmt.Sprintf("user.username == %q", member.Username),
})
writeRule(codersdk.ExperimentMCPToolSearch, experimentrules.Rule{Mode: experimentrules.ModeOff})
requireExperiments(memberClient, "foo", codersdk.ExperimentExample)
requireExperiments(ownerClient, "foo")

// Inherit restores the startup default.
writeRule(codersdk.ExperimentMCPToolSearch, experimentrules.Rule{Mode: experimentrules.ModeInherit})
requireExperiments(memberClient, "foo", codersdk.ExperimentMCPToolSearch, codersdk.ExperimentExample)
requireExperiments(ownerClient, "foo", codersdk.ExperimentMCPToolSearch)

// The personalized result must not be cached.
res, err := memberClient.Request(ctx, http.MethodGet, "/api/v2/experiments", nil)
require.NoError(t, err)
defer res.Body.Close()
require.Equal(t, http.StatusOK, res.StatusCode)
require.Equal(t, "no-store", res.Header.Get("Cache-Control"))
})

t.Run("available experiments", func(t *testing.T) {
t.Parallel()
cfg := coderdtest.DeploymentValues(t)
Expand Down
3 changes: 3 additions & 0 deletions coderd/x/chatd/ARCHITECTURE.md
Original file line number Diff line number Diff line change
Expand Up @@ -823,6 +823,9 @@ The runner maintains the following local state:
- its own `WorkerID` and `RunnerID`.
- a list of goroutines it has spawned to perform side effects, each identified by a unique ID, together with cancellation handles and go channels that the goroutines use to notify they have finished.
- the ID of the currently active goroutine, if there is one.
- per-turn decisions that must hold across all steps of a turn, keyed by the turn's prompt row (the ID of the last user prompt message): currently the chat owner's `mcp-tool-search` experiment decision.

Each step of a turn runs as its own goroutine, so turn-wide decisions live on the runner. The first step with MCP candidates decides `mcp-tool-search` for the chat owner and later steps reuse it, so a rule change applies from the next turn and never withdraws a `find_tools` call already issued. Steps without MCP candidates never offer `find_tools`, so they skip the rule read and cache nothing. A late result from an older turn never replaces a newer decision, because prompt row IDs only increase. Turns without a prompt row are not cached. The decision lives in memory only, so a new runner after a handoff evaluates it again.
Comment thread
ThomasK33 marked this conversation as resolved.
Resolved

### Event processing

Expand Down
15 changes: 12 additions & 3 deletions coderd/x/chatd/chatd.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ import (
"github.com/coder/coder/v2/coderd/database/db2sdk"
"github.com/coder/coder/v2/coderd/database/dbauthz"
"github.com/coder/coder/v2/coderd/database/pubsub"
"github.com/coder/coder/v2/coderd/experiments"
"github.com/coder/coder/v2/coderd/httpmw"
"github.com/coder/coder/v2/coderd/notifications"
coderdpubsub "github.com/coder/coder/v2/coderd/pubsub"
Expand Down Expand Up @@ -200,6 +201,7 @@ type Server struct {

aibridgeTransportFactory *atomic.Pointer[aibridge.TransportFactory]
experiments codersdk.Experiments
experimentEvaluator *experiments.Evaluator

// thinkingDropBlock holds the thinkingDropBlockKey of each provider and
// model that accepted Anthropic's thinking drop_block control.
Expand Down Expand Up @@ -2916,7 +2918,10 @@ type Config struct {
Clock quartz.Clock
AIBridgeTransportFactory *atomic.Pointer[aibridge.TransportFactory]
Experiments codersdk.Experiments
PrometheusRegistry prometheus.Registerer
// ExperimentEvaluator decides user-scoped experiments for a chat's
// owner. It is required.
ExperimentEvaluator *experiments.Evaluator
PrometheusRegistry prometheus.Registerer

AgentCapacityUnlock AgentCapacityUnlock

Expand All @@ -2936,7 +2941,10 @@ type Config struct {
// New creates a new chat processor with the required pubsub dependency.
// The processor polls for pending chats and processes them. It is the
// caller's responsibility to call Close on the returned instance.
func New(ps pubsub.Pubsub, cfg Config) *Server {
func New(ps pubsub.Pubsub, cfg Config) (*Server, error) {
if cfg.ExperimentEvaluator == nil {
return nil, xerrors.New("chatd: experiment evaluator is required")
}
ctx, cancel := context.WithCancel(context.Background())

pendingChatAcquireInterval := cfg.PendingChatAcquireInterval
Expand Down Expand Up @@ -3035,6 +3043,7 @@ func New(ps pubsub.Pubsub, cfg Config) *Server {
},
aibridgeTransportFactory: cfg.AIBridgeTransportFactory,
experiments: cfg.Experiments,
experimentEvaluator: cfg.ExperimentEvaluator,
inFlightChatStaleAfter: inFlightChatStaleAfter,
streamSilenceTimeout: streamSilenceTimeout,
usageTracker: cfg.UsageTracker,
Expand Down Expand Up @@ -3139,7 +3148,7 @@ func New(ps pubsub.Pubsub, cfg Config) *Server {

// Spawn background goroutines that all servers need.

return p
return p, nil
}

// Start runs the background acquire/wake loop that picks up
Expand Down
Loading
Loading