Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions cli/exp_experimentrules.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,15 +26,15 @@ func (r *RootCmd) experimentRulesCommand() *serpent.Command {
FormatExamples(
Example{
Description: "Enable an experiment for users whose condition is true",
Command: `coder exp experiment-rules set mcp-tool-search 'user.email.endsWith("@example.com")'`,
Command: `coder exp experiment-rules set example 'user.email.endsWith("@example.com")'`,
},
Example{
Description: "Turn an experiment off for everyone, even if it is enabled at startup",
Command: "coder exp experiment-rules off mcp-tool-search",
Command: "coder exp experiment-rules off example",
},
Example{
Description: "Restore the startup --experiments default",
Command: "coder exp experiment-rules reset mcp-tool-search",
Command: "coder exp experiment-rules reset example",
},
),
Handler: func(inv *serpent.Invocation) error {
Expand Down
29 changes: 14 additions & 15 deletions cli/exp_experimentrules_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,11 @@ import (
func TestExperimentRules(t *testing.T) {
t.Parallel()

// mcp-tool-search is enabled at startup; example is not.
setup := func(t *testing.T) (*codersdk.Client, *codersdk.Client, func(args ...string) (string, string, error)) {
setup := func(t *testing.T, experiments ...string) (*codersdk.Client, *codersdk.Client, func(args ...string) (string, string, error)) {
t.Helper()
ownerClient := coderdtest.New(t, &coderdtest.Options{
DeploymentValues: coderdtest.DeploymentValues(t, func(v *codersdk.DeploymentValues) {
v.Experiments = []string{string(codersdk.ExperimentMCPToolSearch)}
v.Experiments = experiments
}),
})
owner := coderdtest.CreateFirstUser(t, ownerClient)
Expand Down Expand Up @@ -76,8 +75,9 @@ func TestExperimentRules(t *testing.T) {

_, _, err = run("off", "example")
require.NoError(t, err)
_, _, err = run("reset", "example")
_, stderr, err := run("reset", "example")
require.NoError(t, err)
require.NotContains(t, stderr, "Reset is not a kill switch")
rule := storedRule(ctx, t, ownerClient, codersdk.ExperimentExample)
require.Equal(t, string(codersdk.ExperimentRuleModeInherit), rule.Mode)
require.Equal(t, int64(4), rule.Revision)
Expand All @@ -86,13 +86,12 @@ func TestExperimentRules(t *testing.T) {
require.NoError(t, err)
var entries []codersdk.ExperimentRuleEntry
require.NoError(t, json.Unmarshal([]byte(stdout), &entries))
require.Len(t, entries, 2)
require.Len(t, entries, 1)
require.Equal(t, rule, entries[0].Rule)

stdout, _, err = run("list")
require.NoError(t, err)
require.Regexp(t, `example\s+false\s+inherit\s+4`, stdout)
require.Regexp(t, `mcp-tool-search\s+true\s+\(none\)\s+0`, stdout)
})

// A stale --expected-revision fails with the current state and is not
Expand All @@ -117,19 +116,19 @@ func TestExperimentRules(t *testing.T) {
// that it is not a kill switch.
t.Run("ResetWarnsWhenStaticDefaultOn", func(t *testing.T) {
t.Parallel()
_, memberClient, run := setup(t)
_, memberClient, run := setup(t, string(codersdk.ExperimentExample))
ctx := testutil.Context(t, testutil.WaitLong)

_, _, err := run("off", "mcp-tool-search")
require.NoError(t, err)
requireMemberEnabled(ctx, t, memberClient, codersdk.ExperimentMCPToolSearch, false)
_, stderr, err := run("reset", "mcp-tool-search")
stdout, _, err := run("list")
require.NoError(t, err)
require.Contains(t, stderr, "Reset is not a kill switch")
requireMemberEnabled(ctx, t, memberClient, codersdk.ExperimentMCPToolSearch, true)
require.Regexp(t, `example\s+true\s+\(none\)\s+0`, stdout)

_, stderr, err = run("reset", "example")
_, _, err = run("off", "example")
require.NoError(t, err)
require.NotContains(t, stderr, "Reset is not a kill switch")
requireMemberEnabled(ctx, t, memberClient, codersdk.ExperimentExample, false)
_, stderr, err := run("reset", "example")
require.NoError(t, err)
require.Contains(t, stderr, "Reset is not a kill switch")
requireMemberEnabled(ctx, t, memberClient, codersdk.ExperimentExample, true)
})
}
4 changes: 0 additions & 4 deletions coderd/apidoc/docs.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 0 additions & 4 deletions coderd/apidoc/swagger.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 0 additions & 1 deletion coderd/coderd.go
Original file line number Diff line number Diff line change
Expand Up @@ -946,7 +946,6 @@ func New(options *Options) *API {
StreamSilenceTimeout: streamSilenceTimeout,
DisableCallerSuppliedTools: options.DeploymentValues.DisableChatCallerSuppliedTools.Value(),
Experiments: experiments,
ExperimentEvaluator: api.ExperimentEvaluator,
AgentConn: api.agentProvider.AgentConn,
AgentInactiveDisconnectTimeout: api.AgentInactiveDisconnectTimeout,
CreateWorkspace: api.chatCreateWorkspace,
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
-- no-op. The deleted mcp-tool-search experiment rule is not restored.
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
-- Delete the stored runtime rule for mcp-tool-search, which is not a
-- user-scoped experiment. The server would otherwise keep listing the rule
-- as ignored, and the rules API cannot delete it.
DELETE FROM site_configs WHERE key = 'experiment_rule:mcp-tool-search';
19 changes: 19 additions & 0 deletions coderd/database/migrations/migrate_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -409,6 +409,7 @@ func TestMigrationChain(t *testing.T) {
{"Migration000621Skills", 621, testMigration000621Skills},
{"Migration000625TemplateVersionScriptOrderUsage", 625, testMigration000625TemplateVersionScriptOrderUsage},
{"Migration000635ChatModelOverrideComputerUse", 635, testMigration000635ChatModelOverrideComputerUse},
{"Migration000637DeleteMCPToolSearchExperimentRule", 637, testMigration000637DeleteMCPToolSearchExperimentRule},
}
for _, step := range steps {
stepTo(step.version - 1)
Expand Down Expand Up @@ -4771,3 +4772,21 @@ func testMigration000635ChatModelOverrideComputerUse(t *testing.T, sqlDB *sql.DB
_, err = sqlDB.ExecContext(ctx, string(upSQL))
require.NoError(t, err)
}

func testMigration000637DeleteMCPToolSearchExperimentRule(t *testing.T, sqlDB *sql.DB, next migrationStepper) {
const migrationVersion = 637

ctx := testutil.Context(t, testutil.WaitLong)
_, err := sqlDB.ExecContext(ctx, `INSERT INTO site_configs (key, value) VALUES ('experiment_rule:mcp-tool-search', '{}'), ('experiment_rule:example', '{}')`)
require.NoError(t, err)

version, _, err := next()
require.NoError(t, err)
require.EqualValues(t, migrationVersion, version)

var keys pq.StringArray
err = sqlDB.QueryRowContext(ctx, `SELECT array_agg(key ORDER BY key) FROM site_configs
WHERE key IN ('experiment_rule:mcp-tool-search', 'experiment_rule:example')`).Scan(&keys)
require.NoError(t, err)
require.Equal(t, []string{"experiment_rule:example"}, []string(keys))
}
62 changes: 27 additions & 35 deletions coderd/experimentrules_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -74,13 +74,11 @@ func ruleEntry(ctx context.Context, t *testing.T, client *codersdk.ExperimentalC
func TestExperimentRules(t *testing.T) {
t.Parallel()

// Each write takes effect on the member's next request, without a
// restart. mcp-tool-search is enabled at startup, example is not.
t.Run("WritesApplyToNextRequest", func(t *testing.T) {
t.Parallel()
ownerClient := coderdtest.New(t, &coderdtest.Options{
DeploymentValues: coderdtest.DeploymentValues(t, func(v *codersdk.DeploymentValues) {
v.Experiments = []string{string(codersdk.ExperimentMCPToolSearch)}
v.Experiments = []string{string(codersdk.ExperimentExample)}
}),
})
owner := coderdtest.CreateFirstUser(t, ownerClient)
Expand All @@ -91,32 +89,28 @@ func TestExperimentRules(t *testing.T) {
entries, err := w.client.ExperimentRules(ctx)
require.NoError(t, err)
require.Equal(t, []codersdk.ExperimentRuleEntry{
{Experiment: string(codersdk.ExperimentExample)},
{Experiment: string(codersdk.ExperimentMCPToolSearch), StaticDefault: true},
{Experiment: string(codersdk.ExperimentExample), StaticDefault: true},
}, entries)

w.put(ctx, codersdk.ExperimentExample, codersdk.ExperimentRuleModeOn, "")
requireEnabled(ctx, t, memberClient, codersdk.ExperimentExample, true)

w.put(ctx, codersdk.ExperimentExample, codersdk.ExperimentRuleModeCondition, fmt.Sprintf("user.username == %q", member.Username))
requireEnabled(ctx, t, memberClient, codersdk.ExperimentExample, true)
requireEnabled(ctx, t, ownerClient, codersdk.ExperimentExample, false)

w.put(ctx, codersdk.ExperimentExample, codersdk.ExperimentRuleModeOff, "")
requireEnabled(ctx, t, memberClient, codersdk.ExperimentExample, false)
w.put(ctx, codersdk.ExperimentExample, codersdk.ExperimentRuleModeOn, "")
requireEnabled(ctx, t, ownerClient, codersdk.ExperimentExample, true)

// off is a kill switch for a statically enabled experiment, and
// reset restores the startup default.
w.put(ctx, codersdk.ExperimentMCPToolSearch, codersdk.ExperimentRuleModeOff, "")
requireEnabled(ctx, t, memberClient, codersdk.ExperimentMCPToolSearch, false)
w.put(ctx, codersdk.ExperimentMCPToolSearch, codersdk.ExperimentRuleModeInherit, "")
requireEnabled(ctx, t, memberClient, codersdk.ExperimentMCPToolSearch, true)
w.put(ctx, codersdk.ExperimentExample, codersdk.ExperimentRuleModeOff, "")
requireEnabled(ctx, t, memberClient, codersdk.ExperimentExample, false)
w.put(ctx, codersdk.ExperimentExample, codersdk.ExperimentRuleModeInherit, "")
requireEnabled(ctx, t, memberClient, codersdk.ExperimentExample, true)

entry := ruleEntry(ctx, t, w.client, codersdk.ExperimentMCPToolSearch)
entry := ruleEntry(ctx, t, w.client, codersdk.ExperimentExample)
require.True(t, entry.StaticDefault)
require.NotNil(t, entry.Rule)
require.Equal(t, string(codersdk.ExperimentRuleModeInherit), entry.Rule.Mode)
require.Equal(t, int64(2), entry.Rule.Revision)
require.Equal(t, int64(4), entry.Rule.Revision)
require.Equal(t, owner.UserID, entry.Rule.UpdatedBy)
require.False(t, entry.Rule.UpdatedAt.IsZero())

Expand All @@ -126,7 +120,7 @@ func TestExperimentRules(t *testing.T) {
body any
}{
{http.MethodGet, "/api/experimental/experiments/rules", nil},
{http.MethodPut, "/api/experimental/experiments/rules/example", codersdk.PutExperimentRuleRequest{Mode: codersdk.ExperimentRuleModeOff, ExpectedRevision: 3}},
{http.MethodPut, "/api/experimental/experiments/rules/example", codersdk.PutExperimentRuleRequest{Mode: codersdk.ExperimentRuleModeInherit, ExpectedRevision: 4}},
} {
res, err := ownerClient.Request(ctx, req.method, req.path, req.body)
require.NoError(t, err)
Expand Down Expand Up @@ -176,7 +170,6 @@ func TestExperimentRules(t *testing.T) {
require.NoError(t, err)
require.Equal(t, []codersdk.ExperimentRuleEntry{
{Experiment: string(codersdk.ExperimentExample), Rule: &stored},
{Experiment: string(codersdk.ExperimentMCPToolSearch)},
}, entries)
})

Expand Down Expand Up @@ -220,21 +213,23 @@ func TestExperimentRules(t *testing.T) {
memberClient, _ := coderdtest.CreateAnotherUser(t, ownerClient, owner.OrganizationID)
ctx := testutil.Context(t, testutil.WaitLong)
sysCtx := dbauthz.AsSystemRestricted(ctx)
store := func(ex codersdk.Experiment, value string) {
t.Helper()
require.NoError(t, db.UpsertExperimentRule(sysCtx, database.UpsertExperimentRuleParams{Experiment: string(ex), Value: value}))
}
for ex, value := range map[codersdk.Experiment]string{
"not-an-experiment": `{"mode":"on","revision":1}`,
codersdk.ExperimentWorkspaceUsage: `{"mode":"off","revision":4}`,
codersdk.ExperimentExample: `{"mode":"percent","revision":2}`,
codersdk.ExperimentMCPToolSearch: `{not json`,
} {
require.NoError(t, db.UpsertExperimentRule(sysCtx, database.UpsertExperimentRuleParams{Experiment: string(ex), Value: value}))
store(ex, value)
}
client := codersdk.NewExperimentalClient(ownerClient)

entries, err := client.ExperimentRules(ctx)
require.NoError(t, err)
require.Equal(t, []codersdk.ExperimentRuleEntry{
{Experiment: string(codersdk.ExperimentExample), Rule: &codersdk.ExperimentRule{Revision: 2}},
{Experiment: string(codersdk.ExperimentMCPToolSearch), Rule: &codersdk.ExperimentRule{}},
{Experiment: "not-an-experiment", Rule: &codersdk.ExperimentRule{Mode: string(codersdk.ExperimentRuleModeOn), Revision: 1}, Ignored: true},
{Experiment: string(codersdk.ExperimentWorkspaceUsage), Rule: &codersdk.ExperimentRule{Mode: string(codersdk.ExperimentRuleModeOff), Revision: 4}, Ignored: true},
}, entries)
Expand All @@ -243,35 +238,32 @@ func TestExperimentRules(t *testing.T) {
require.NoError(t, err)
require.Equal(t, int64(3), rule.Revision)

_, err = client.PutExperimentRule(ctx, codersdk.ExperimentMCPToolSearch, codersdk.PutExperimentRuleRequest{Mode: codersdk.ExperimentRuleModeOn, ExpectedRevision: 5})
store(codersdk.ExperimentExample, `{not json`)
require.Equal(t, &codersdk.ExperimentRule{}, ruleEntry(ctx, t, client, codersdk.ExperimentExample).Rule)
_, err = client.PutExperimentRule(ctx, codersdk.ExperimentExample, codersdk.PutExperimentRuleRequest{Mode: codersdk.ExperimentRuleModeOn, ExpectedRevision: 5})
requireStatus(t, err, http.StatusConflict)
rule, err = client.PutExperimentRule(ctx, codersdk.ExperimentMCPToolSearch, codersdk.PutExperimentRuleRequest{Mode: codersdk.ExperimentRuleModeOn, ExpectedRevision: 0})
rule, err = client.PutExperimentRule(ctx, codersdk.ExperimentExample, codersdk.PutExperimentRuleRequest{Mode: codersdk.ExperimentRuleModeOn, ExpectedRevision: 0})
require.NoError(t, err)
require.Equal(t, int64(1), rule.Revision)
for _, c := range []*codersdk.Client{ownerClient, memberClient} {
requireEnabled(ctx, t, c, codersdk.ExperimentExample, true)
requireEnabled(ctx, t, c, codersdk.ExperimentMCPToolSearch, true)
}

// An on rule without a positive revision is malformed: it is
// listed at revision 0 and decides off for everyone.
for ex, value := range map[codersdk.Experiment]string{
codersdk.ExperimentExample: `{"mode":"on","revision":0}`,
codersdk.ExperimentMCPToolSearch: `{"mode":"on"}`,
} {
require.NoError(t, db.UpsertExperimentRule(sysCtx, database.UpsertExperimentRuleParams{Experiment: string(ex), Value: value}))
require.Equal(t, &codersdk.ExperimentRule{}, ruleEntry(ctx, t, client, ex).Rule, ex)
}
for _, c := range []*codersdk.Client{ownerClient, memberClient} {
requireEnabled(ctx, t, c, codersdk.ExperimentExample, false)
requireEnabled(ctx, t, c, codersdk.ExperimentMCPToolSearch, false)
for _, value := range []string{`{"mode":"on","revision":0}`, `{"mode":"on"}`} {
store(codersdk.ExperimentExample, value)
require.Equal(t, &codersdk.ExperimentRule{}, ruleEntry(ctx, t, client, codersdk.ExperimentExample).Rule, value)
for _, c := range []*codersdk.Client{ownerClient, memberClient} {
requireEnabled(ctx, t, c, codersdk.ExperimentExample, false)
}
}
})
}

// TestExperimentRulesReplicas checks that a rule written through one
// replica decides the next evaluation on another replica sharing the
// database, both for the API and for the evaluator chatd uses.
// database, both for the API and for server checks through the evaluator.
func TestExperimentRulesReplicas(t *testing.T) {
t.Parallel()
db, ps := dbtestutil.NewDB(t)
Expand Down
11 changes: 6 additions & 5 deletions coderd/experiments_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,7 @@ func Test_Experiments(t *testing.T) {
t.Run("rules per user", func(t *testing.T) {
t.Parallel()
cfg := coderdtest.DeploymentValues(t)
cfg.Experiments = []string{"foo", string(codersdk.ExperimentMCPToolSearch)}
cfg.Experiments = []string{"foo", string(codersdk.ExperimentExample)}
ownerClient, db := coderdtest.NewWithDatabase(t, &coderdtest.Options{
DeploymentValues: cfg,
})
Expand Down Expand Up @@ -154,14 +154,15 @@ func Test_Experiments(t *testing.T) {
Mode: experimentrules.ModeCondition,
Condition: fmt.Sprintf("user.username == %q", member.Username),
})
writeRule(codersdk.ExperimentMCPToolSearch, experimentrules.Rule{Mode: experimentrules.ModeOff})
requireExperiments(memberClient, "foo", codersdk.ExperimentExample)
requireExperiments(ownerClient, "foo")
writeRule(codersdk.ExperimentExample, experimentrules.Rule{Mode: experimentrules.ModeOff})
requireExperiments(memberClient, "foo")

// Inherit restores the startup default.
writeRule(codersdk.ExperimentMCPToolSearch, experimentrules.Rule{Mode: experimentrules.ModeInherit})
requireExperiments(memberClient, "foo", codersdk.ExperimentMCPToolSearch, codersdk.ExperimentExample)
requireExperiments(ownerClient, "foo", codersdk.ExperimentMCPToolSearch)
writeRule(codersdk.ExperimentExample, experimentrules.Rule{Mode: experimentrules.ModeInherit})
requireExperiments(memberClient, "foo", codersdk.ExperimentExample)
requireExperiments(ownerClient, "foo", codersdk.ExperimentExample)

// The personalized result must not be cached.
res, err := memberClient.Request(ctx, http.MethodGet, "/api/v2/experiments", nil)
Expand Down
Loading
Loading