Repository navigation
feat: add reverse-proxy handler to AI Gateway #29656
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
09d2561
1a30fde
f7cdbcd
96b98e7
f4310a5
71af097
32c1808
914fbba
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -6,21 +6,18 @@ import ( | |
| "fmt" | ||
| "net/http" | ||
| "net/url" | ||
| "strconv" | ||
| "strings" | ||
| "sync" | ||
| "time" | ||
|
|
||
| "github.com/google/uuid" | ||
| "github.com/hashicorp/go-multierror" | ||
| "github.com/sony/gobreaker/v2" | ||
| "go.opentelemetry.io/otel/codes" | ||
| "go.opentelemetry.io/otel/trace" | ||
| "golang.org/x/net/http/httpguts" | ||
| "golang.org/x/xerrors" | ||
|
|
||
| "cdr.dev/slog/v3" | ||
| "github.com/coder/coder/v2/aibridge/circuitbreaker" | ||
| "github.com/coder/coder/v2/aibridge/clientmeta" | ||
| aibcontext "github.com/coder/coder/v2/aibridge/context" | ||
| "github.com/coder/coder/v2/aibridge/intercept" | ||
| "github.com/coder/coder/v2/aibridge/mcp" | ||
|
|
@@ -29,7 +26,6 @@ import ( | |
| "github.com/coder/coder/v2/aibridge/recorder" | ||
| "github.com/coder/coder/v2/aibridge/routing" | ||
| "github.com/coder/coder/v2/aibridge/tracing" | ||
| agplaibridge "github.com/coder/coder/v2/coderd/aibridge" | ||
| "github.com/coder/quartz" | ||
| ) | ||
|
|
||
|
|
@@ -79,39 +75,26 @@ func NewRequestBridge(ctx context.Context, providers []provider.Provider, rec re | |
| continue | ||
| } | ||
|
|
||
| // Create per-provider circuit breaker if configured | ||
| cfg := prov.CircuitBreakerConfig() | ||
| providerName := prov.Name() | ||
| onChange := func(endpoint, model string, from, to gobreaker.State) { | ||
| logger.Info(context.Background(), "circuit breaker state change", | ||
| slog.F("provider", providerName), | ||
| slog.F("endpoint", endpoint), | ||
| slog.F("model", model), | ||
| slog.F("from", from.String()), | ||
| slog.F("to", to.String()), | ||
| ) | ||
| if m != nil { | ||
| m.CircuitBreakerState.WithLabelValues(providerName, endpoint, model).Set(circuitbreaker.StateToGaugeValue(to)) | ||
| if to == gobreaker.StateOpen { | ||
| m.CircuitBreakerTrips.WithLabelValues(providerName, endpoint, model).Inc() | ||
| } | ||
| } | ||
| } | ||
| cbs := circuitbreaker.NewProviderCircuitBreakers(providerName, cfg, onChange, m) | ||
| cbs := circuitbreaker.NewProviderCircuitBreakersWithObservability( | ||
| prov.Name(), prov.CircuitBreakerConfig(), logger, m, | ||
| ) | ||
|
|
||
| // Add the known provider-specific routes which are bridged (i.e. intercepted and augmented). | ||
| for _, path := range prov.BridgedRoutes() { | ||
| handler := newInterceptionProcessor(prov, cbs, rec, mcpProxy, logger, m, tracer) | ||
| route, err := url.JoinPath(prov.RoutePrefix(), path) | ||
| if err != nil { | ||
| logger.Error(ctx, "failed to join path", | ||
| slog.Error(err), | ||
| slog.F("provider", providerName), | ||
| slog.F("provider", prov.Name()), | ||
| slog.F("prefix", prov.RoutePrefix()), | ||
| slog.F("path", path), | ||
| ) | ||
| return nil, xerrors.Errorf("failed to configure provider '%v': failed to join bridged path: %w", providerName, err) | ||
| return nil, xerrors.Errorf("failed to configure provider '%v': failed to join bridged path: %w", prov.Name(), err) | ||
| } | ||
| handler := newInterceptionProcessor( | ||
| prov, cbs, rec, mcpProxy, logger, m, tracer, | ||
| strings.TrimPrefix(route, "/"+prov.Name()), | ||
| ) | ||
| mux.Handle(route, handler) | ||
| } | ||
|
|
||
|
|
@@ -125,11 +108,11 @@ func NewRequestBridge(ctx context.Context, providers []provider.Provider, rec re | |
| if err != nil { | ||
| logger.Error(ctx, "failed to join path", | ||
| slog.Error(err), | ||
| slog.F("provider", providerName), | ||
| slog.F("provider", prov.Name()), | ||
| slog.F("prefix", prov.RoutePrefix()), | ||
| slog.F("path", path), | ||
| ) | ||
| return nil, xerrors.Errorf("failed to configure provider '%v': failed to join passed through path: %w", providerName, err) | ||
| return nil, xerrors.Errorf("failed to configure provider '%v': failed to join passed through path: %w", prov.Name(), err) | ||
| } | ||
| mux.Handle(route, http.StripPrefix(prov.RoutePrefix(), ftr)) | ||
| } | ||
|
|
@@ -156,7 +139,7 @@ func WithClock(clock quartz.Clock) RequestBridgeOption { | |
| // newInterceptionProcessor returns an [http.HandlerFunc] which is capable of creating a new interceptor and processing a given request | ||
| // using [Provider] p, recording all usage events using [Recorder] rec. | ||
| // If cbs is non-nil, circuit breaker protection is applied per endpoint/model tuple. | ||
| func newInterceptionProcessor(p provider.Provider, cbs *circuitbreaker.ProviderCircuitBreakers, rec recorder.Recorder, mcpProxy mcp.ServerProxier, logger slog.Logger, m *metrics.Metrics, tracer trace.Tracer) http.HandlerFunc { | ||
| func newInterceptionProcessor(p provider.Provider, cbs *circuitbreaker.ProviderCircuitBreakers, rec recorder.Recorder, mcpProxy mcp.ServerProxier, logger slog.Logger, m *metrics.Metrics, tracer trace.Tracer, route string) http.HandlerFunc { | ||
| return func(w http.ResponseWriter, r *http.Request) { | ||
| ctx, span := tracer.Start(r.Context(), "Intercept") | ||
| defer span.End() | ||
|
|
@@ -167,7 +150,6 @@ func newInterceptionProcessor(p provider.Provider, cbs *circuitbreaker.ProviderC | |
| sessionID := GuessSessionID(client, r) | ||
|
|
||
| if isWebSocketUpgrade(r) { | ||
| route := strings.TrimPrefix(r.URL.Path, fmt.Sprintf("/%s", p.Name())) | ||
| logger.Debug(ctx, "rejecting unsupported WebSocket upgrade", | ||
| slog.F("provider", p.Name()), | ||
| slog.F("route", route), | ||
|
|
@@ -259,7 +241,6 @@ func newInterceptionProcessor(p provider.Provider, cbs *circuitbreaker.ProviderC | |
| return | ||
| } | ||
|
|
||
| route := strings.TrimPrefix(r.URL.Path, fmt.Sprintf("/%s", p.Name())) | ||
| log := logger.With( | ||
| slog.F("route", route), | ||
| slog.F("provider", p.Name()), | ||
|
|
@@ -288,13 +269,13 @@ func newInterceptionProcessor(p provider.Provider, cbs *circuitbreaker.ProviderC | |
| errType, errMsg := categorizeInterceptionError(p, execErr) | ||
| if execErr != nil { | ||
| if m != nil { | ||
| m.InterceptionCount.WithLabelValues(p.Name(), interceptor.Model(), metrics.InterceptionCountStatusFailed, route, r.Method, actor.ID, string(client)).Add(1) | ||
| m.InterceptionCount.WithLabelValues(p.Name(), interceptor.Model(), metrics.InterceptionCountStatusFailed, route, routing.MetricMethod(r.Method), actor.ID, string(client)).Add(1) | ||
| } | ||
| span.SetStatus(codes.Error, fmt.Sprintf("interception failed: %v", execErr)) | ||
| log.Warn(credCtx, "interception failed", slog.Error(execErr), slog.F("error_type", string(errType))) | ||
| } else { | ||
| if m != nil { | ||
| m.InterceptionCount.WithLabelValues(p.Name(), interceptor.Model(), metrics.InterceptionCountStatusCompleted, route, r.Method, actor.ID, string(client)).Add(1) | ||
| m.InterceptionCount.WithLabelValues(p.Name(), interceptor.Model(), metrics.InterceptionCountStatusCompleted, route, routing.MetricMethod(r.Method), actor.ID, string(client)).Add(1) | ||
| } | ||
| log.Debug(credCtx, "interception ended") | ||
| } | ||
|
|
@@ -313,9 +294,7 @@ func newInterceptionProcessor(p provider.Provider, cbs *circuitbreaker.ProviderC | |
|
|
||
| // isWebSocketUpgrade reports whether r is a WebSocket opening handshake. | ||
| func isWebSocketUpgrade(r *http.Request) bool { | ||
| return r.Method == http.MethodGet && | ||
| httpguts.HeaderValuesContainsToken(r.Header.Values("Connection"), "upgrade") && | ||
| httpguts.HeaderValuesContainsToken(r.Header.Values("Upgrade"), "websocket") | ||
| return clientmeta.IsWebSocketUpgrade(r) | ||
| } | ||
|
|
||
| // ServeHTTP exposes the internal http.Handler, which has all [Provider]s' routes registered. | ||
|
|
@@ -347,44 +326,8 @@ func (b *RequestBridge) Shutdown(ctx context.Context) error { | |
| return err | ||
| } | ||
|
|
||
| // extractAgentFirewallHeaders reads and parses the Agent Firewall | ||
| // correlation headers from the request. Both headers must be present | ||
| // together with a valid UUID session ID and a non-negative int32 | ||
| // sequence number, or both must be absent. Partial or malformed headers | ||
| // return an error so the caller can reject the request (fail closed). | ||
| func extractAgentFirewallHeaders(r *http.Request) (sessionID *string, seqNumber *int32, err error) { | ||
| rawSessionID := r.Header.Get(agplaibridge.HeaderAgentFirewallSessionID) | ||
| rawSeqNumber := r.Header.Get(agplaibridge.HeaderAgentFirewallSequenceNumber) | ||
|
|
||
| hasSessionID := rawSessionID != "" | ||
| hasSeqNumber := rawSeqNumber != "" | ||
|
|
||
| switch { | ||
| case !hasSessionID && !hasSeqNumber: | ||
| // Neither header present; request did not traverse Agent Firewall. | ||
| return nil, nil, nil | ||
| case hasSessionID && !hasSeqNumber: | ||
| return nil, nil, xerrors.Errorf("agent firewall session ID header present without sequence number") | ||
| case !hasSessionID && hasSeqNumber: | ||
| return nil, nil, xerrors.Errorf("agent firewall sequence number header present without session ID") | ||
| } | ||
|
|
||
| // Both headers present; validate the session ID is a UUID. Storing an | ||
| // invalid value would silently drop the firewall correlation to NULL | ||
| // downstream, so reject it here instead. | ||
| if _, parseErr := uuid.Parse(rawSessionID); parseErr != nil { | ||
| return nil, nil, xerrors.Errorf("invalid agent firewall session ID %q: %w", rawSessionID, parseErr) | ||
| } | ||
|
|
||
| // Parse the sequence number. | ||
| n, err := strconv.ParseInt(rawSeqNumber, 10, 32) | ||
| if err != nil { | ||
| return nil, nil, xerrors.Errorf("invalid agent firewall sequence number %q: %w", rawSeqNumber, err) | ||
| } | ||
| if n < 0 { | ||
| return nil, nil, xerrors.Errorf("invalid agent firewall sequence number %q: must be non-negative", rawSeqNumber) | ||
| } | ||
|
|
||
| n32 := int32(n) | ||
| return &rawSessionID, &n32, nil | ||
| // extractAgentFirewallHeaders returns validated Agent Firewall correlation | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. P3 [CRF-109] Two unexported single-call wrappers over
These are unexported, so unlike the Gon reached the second one from the comment side and found the related defect: "returns validated Agent Firewall correlation metadata without exposing raw malformed values in errors" is a claim about the callee's invariant, which is implemented and documented in
If
|
||
| // metadata without exposing raw malformed values in errors. | ||
| func extractAgentFirewallHeaders(r *http.Request) (*string, *int32, error) { | ||
| return clientmeta.ExtractAgentFirewallHeaders(r) | ||
| } | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
P2 [CRF-83] The CRF-35 fix bounded the method and route labels on this line and left
interceptor.Model(), a raw client string, as a label on three metric families and half a never-evicted map key. (Killua P2, Mafu-san P2)Re-raise gate: round 3 recorded this (Mafuuu, P4) and dropped it from posting because it predated the diff, with the note that round 4 may raise it if the diff touches it. Round 4 touches it. This commit rewrote this exact expression, turning
r.Methodintorouting.MetricMethod(r.Method)on lines 272 and 278 and precomputingrouteat mux registration, and leftinterceptor.Model()alone in the same argument list.aibridge/bridge.goalso now builds its breakers through this PR's new constructor, which makes it the shared construction path for both modes.I verified the retention path rather than the measurement:
circuitbreaker.go:103buildskey := endpoint + ":" + model, stores it withLoadOrStorein async.Map, and there is no delete anywhere in the file. The breaker is also created before the upstream call, so a model the provider would reject still allocates a permanent entry. The metric half is unconditional wherever coderd supplies a registry; the breaker half needs a non-nilCircuitBreakerConfig().This is the interception path serving production today, and any authenticated user can drive it. Fix: bound
modelthe wayMetricMethodbounds the method, with an allowlist of configured or known model identifiers and oneotherfallback. That closes the label and the breaker key together, and it is smaller than the change that introducedMetricMethod. Proxy mode is clean here; it passes the empty model, which is CRF-93.