Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions agent/api.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,11 @@ func (a *agent) apiHandler() http.Handler {
r.Use(
httpmw.Recover(a.logger),
tracing.StatusWriterMiddleware,
// Reuse the coderd tracing middleware with a noop tracer (nil provider):
// it emits no spans or telemetry on the agent and only enriches the log
// context with client_session_id. Tracks the API, debug (for
// support-bundle correlation), and root routes.
tracing.Middleware(nil, []string{"/", "/api", "/api/**", "/debug/**"}, "agent"),
loggermw.Logger(a.logger, nil),
agentchat.Middleware,
)
Expand Down
58 changes: 58 additions & 0 deletions agent/api_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
package agent_test

import (
"net/http"
"testing"

"github.com/stretchr/testify/require"

"cdr.dev/slog/v3"
"github.com/coder/coder/v2/agent"
"github.com/coder/coder/v2/agent/agenttest"
"github.com/coder/coder/v2/coderd/tracing"
"github.com/coder/coder/v2/codersdk/agentsdk"
"github.com/coder/coder/v2/testutil"
)

// TestAgent_APIClientSessionID verifies that a client_session_id set as baggage
// on the agent connection reaches the agent's request logs. workspacesdk builds
// a separate HTTP client per agent API request, so the CLI's baggage transport
// wrapper does not apply; the ID rides along via the connection's extra headers
// instead. The agent's tracing middleware reads client_session_id from the
// baggage header and adds it to the request log context.
func TestAgent_APIClientSessionID(t *testing.T) {
t.Parallel()

const sessionID = "0123456789abcdef0123456789abcdef"
ctx := testutil.Context(t, testutil.WaitLong)

sink := testutil.NewFakeSink(t)

//nolint:dogsled
conn, _, _, _, _ := setupAgent(t, agentsdk.Manifest{}, 0, func(_ *agenttest.Client, o *agent.Options) {
o.Logger = sink.Logger().Named("agent")
})
require.True(t, conn.AwaitReachable(ctx))

conn.SetExtraHeaders(http.Header{
"baggage": []string{tracing.SessionIDBaggageKey + "=" + sessionID},
})

// Any agent HTTP API call exercises the per-request client Asher flagged;
// ListeningPorts hits /api/v0/listening-ports, which the agent middleware
// tracks.
_, err := conn.ListeningPorts(ctx)
require.NoError(t, err)

// The middleware should have added client_session_id to the request log
// context, so at least one agent log entry carries it.
entries := sink.Entries(func(e slog.SinkEntry) bool {
for _, f := range e.Fields {
if f.Name == "client_session_id" {
return f.Value == sessionID
}
}
return false
})
require.NotEmpty(t, entries, "client_session_id should reach the agent request logs")
}
2 changes: 1 addition & 1 deletion coderd/coderd.go
Original file line number Diff line number Diff line change
Expand Up @@ -1149,7 +1149,7 @@ func New(options *Options) *API {
httpmw.WithProfilingLabels,
tracing.StatusWriterMiddleware,
options.DeploymentValues.HTTPCookies.Middleware,
tracing.Middleware(api.TracerProvider),
tracing.Middleware(api.TracerProvider, tracing.DefaultRoutePatterns, "coderd"),
httpmw.AttachRequestID,
httpmw.ExtractRealIP(api.RealIPConfig),
loggermw.Logger(api.Logger, func(r *http.Request) string {
Expand Down
40 changes: 25 additions & 15 deletions coderd/tracing/httpmw.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,18 +27,27 @@ import (
// a 16-byte identifier encoded as a 32-character hexadecimal string.
const SessionIDBaggageKey = "client_session_id"

// Middleware adds tracing to http routes.
func Middleware(tracerProvider trace.TracerProvider) func(http.Handler) http.Handler {
// We only want to create spans on the following route patterns, however
// we want the middleware to be very high in the middleware stack so it can
// capture the entire request.
re := patternmatcher.RoutePatterns{
"/api",
"/api/**",
"/@*/*/apps/**",
"/%40*/*/apps/**",
"/external-auth/*/callback",
}.MustCompile()
// DefaultRoutePatterns are the route patterns coderd and wsproxy trace. The
// middleware runs high in the stack so it can capture the entire request, but
// only acts on these patterns.
var DefaultRoutePatterns = []string{
"/api",
"/api/**",
"/@*/*/apps/**",
"/%40*/*/apps/**",
"/external-auth/*/callback",
}

// Middleware adds tracing to http routes. It only acts on requests matching
// routePatterns, and labels emitted spans with serverName. A nil tracerProvider
// disables span creation, leaving the middleware to enrich the log context with
// client_session_id only (as the agent uses it).
func Middleware(
tracerProvider trace.TracerProvider,
routePatterns []string,
serverName string,
) func(http.Handler) http.Handler {
re := patternmatcher.RoutePatterns(routePatterns).MustCompile()

if tracerProvider == nil {
tracerProvider = noop.NewTracerProvider()
Expand Down Expand Up @@ -80,7 +89,7 @@ func Middleware(tracerProvider trace.TracerProvider) func(http.Handler) http.Han
// pass the span through the request context and serve the request to the next middleware
next.ServeHTTP(sw, r)
// capture response data
EndHTTPSpan(r, sw.Status, span)
EndHTTPSpan(r, sw.Status, span, serverName)
})
}
}
Expand Down Expand Up @@ -160,12 +169,13 @@ func StartHTTPSpan(tracer trace.Tracer, rw http.ResponseWriter, r *http.Request,
}

// EndHTTPSpan captures request and response data after the handler is done.
func EndHTTPSpan(r *http.Request, status int, span trace.Span) {
// serverName labels the span's server request attributes.
func EndHTTPSpan(r *http.Request, status int, span trace.Span, serverName string) {
// set the resource name as we get it only once the handler is executed
route := chi.RouteContext(r.Context()).RoutePattern()
span.SetName(fmt.Sprintf("%s %s", r.Method, route))
span.SetAttributes(netconv.Transport("tcp"))
span.SetAttributes(httpconv.ServerRequest("coderd", r)...)
span.SetAttributes(httpconv.ServerRequest(serverName, r)...)
span.SetAttributes(semconv.HTTPRouteKey.String(route))

// 0 status means one has not yet been sent in which case net/http library will write StatusOK
Expand Down
168 changes: 164 additions & 4 deletions coderd/tracing/httpmw_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (
"go.opentelemetry.io/otel/trace/noop"

"cdr.dev/slog/v3"
"github.com/coder/coder/v2/coderd/httpmw/loggermw"
"github.com/coder/coder/v2/coderd/tracing"
"github.com/coder/coder/v2/testutil"
)
Expand Down Expand Up @@ -99,7 +100,7 @@ func Test_Middleware_SessionID(t *testing.T) {
ctx := context.WithValue(context.Background(), chi.RouteCtxKey, chi.NewRouteContext())
r = r.WithContext(ctx)

tracing.Middleware(tp)(handler).ServeHTTP(rw, r)
tracing.Middleware(tp, tracing.DefaultRoutePatterns, "coderd")(handler).ServeHTTP(rw, r)

entries := sink.Entries(func(e slog.SinkEntry) bool {
return e.Message == "downstream handler invoked"
Expand Down Expand Up @@ -289,7 +290,7 @@ func Test_Middleware_SessionID(t *testing.T) {
ctx = context.WithValue(ctx, chi.RouteCtxKey, chi.NewRouteContext())
r = r.WithContext(ctx)

tracing.Middleware(provider)(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
tracing.Middleware(provider, tracing.DefaultRoutePatterns, "coderd")(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
rw.WriteHeader(http.StatusOK)
})).ServeHTTP(rw, r)

Expand Down Expand Up @@ -342,6 +343,165 @@ func (*startNameRecorder) OnEnd(sdktrace.ReadOnlySpan) {}
func (*startNameRecorder) Shutdown(context.Context) error { return nil }
func (*startNameRecorder) ForceFlush(context.Context) error { return nil }

// Test_Middleware_Agent verifies the agent's reuse of Middleware: with a nil
// tracer it emits no spans and only enriches the log context with
// client_session_id, gated to the agent's /api patterns. This mirrors
// agent/api.go.
func Test_Middleware_Agent(t *testing.T) {
t.Parallel()

agentPatterns := []string{"/", "/api", "/api/**", "/debug/**"}

// accessLogFields runs a request through the agent middleware stack and
// returns the fields on loggermw's request completion log line.
accessLogFields := func(t *testing.T, path string) []slog.Field {
t.Helper()

sink := testutil.NewFakeSink(t)

// StatusWriterMiddleware is required by loggermw; Middleware runs before
// loggermw so the field is on the request context when the access log is
// emitted. A nil tracer means no spans, matching the agent. This mirrors
// agent/api.go.
handler := tracing.StatusWriterMiddleware(
tracing.Middleware(nil, agentPatterns, "agent")(
loggermw.Logger(sink.Logger(), nil)(
http.HandlerFunc(func(rw http.ResponseWriter, _ *http.Request) {
rw.WriteHeader(http.StatusNoContent)
}),
),
),
)

r := httptest.NewRequest(http.MethodGet, path, nil)
r.Header.Set("baggage", tracing.SessionIDBaggageKey+"="+testSessionID)
ctx := context.WithValue(r.Context(), chi.RouteCtxKey, chi.NewRouteContext())
handler.ServeHTTP(httptest.NewRecorder(), r.WithContext(ctx))

entries := sink.Entries()
require.Len(t, entries, 1)
return entries[0].Fields
}

sessionIDField := func(fields []slog.Field) (any, bool) {
for _, f := range fields {
if f.Name == "client_session_id" {
return f.Value, true
}
}
return nil, false
}

t.Run("APIRouteLogged", func(t *testing.T) {
t.Parallel()

val, ok := sessionIDField(accessLogFields(t, "/api/v0/foo"))
require.True(t, ok, "client_session_id should be on the agent access log")
require.Equal(t, testSessionID, val)
})

t.Run("DebugAndRootRoutesLogged", func(t *testing.T) {
t.Parallel()

// The agent also tracks its debug endpoints (for support-bundle
// correlation) and root handler.
for _, path := range []string{"/", "/debug/logs", "/debug/magicsock"} {
val, ok := sessionIDField(accessLogFields(t, path))
require.True(t, ok, "client_session_id should be logged on %q", path)
require.Equal(t, testSessionID, val)
}
})

t.Run("UnmatchedRouteGated", func(t *testing.T) {
t.Parallel()

// A path outside the tracked patterns still gets no client_session_id,
// even with valid baggage.
_, ok := sessionIDField(accessLogFields(t, "/foo"))
require.False(t, ok, "client_session_id must not be logged on an unmatched route")
})
}

// Test_Middleware_CustomRoutePatterns verifies the routePatterns argument gates
// the middleware instead of the coderd defaults.
func Test_Middleware_CustomRoutePatterns(t *testing.T) {
t.Parallel()

cases := []struct {
path string
runs bool
}{
{"/api", true},
{"/api/v0/foo", true},
{"/debug/logs", false},
{"/", false},
// A coderd-only default pattern must not match under agent patterns.
{"/external-auth/hi/callback", false},
}

for _, c := range cases {
name := strings.ReplaceAll(strings.TrimPrefix(c.path, "/"), "/", "_")
if name == "" {
name = "root"
}
t.Run(name, func(t *testing.T) {
t.Parallel()

fake := &fakeTracer{}
rw := &tracing.StatusWriter{ResponseWriter: httptest.NewRecorder()}
r := httptest.NewRequest("GET", c.path, nil)

ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
defer cancel()
ctx = context.WithValue(ctx, chi.RouteCtxKey, chi.NewRouteContext())
r = r.WithContext(ctx)

tracing.Middleware(fake, []string{"/api", "/api/**"}, "agent")(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
rw.WriteHeader(http.StatusNoContent)
})).ServeHTTP(rw, r)

didRun := fake.startCalled.Load() == 1
require.Equal(t, c.runs, didRun, "expected middleware to run/not run")
})
}
}

// Test_Middleware_ServerName verifies the configured server name reaches the
// emitted span via httpconv.ServerRequest.
func Test_Middleware_ServerName(t *testing.T) {
t.Parallel()

recorder := tracetest.NewSpanRecorder()
provider := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(recorder))

rw := &tracing.StatusWriter{ResponseWriter: httptest.NewRecorder()}
r := httptest.NewRequest(http.MethodGet, "/api/v2/workspaces", nil)

ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
defer cancel()
ctx = context.WithValue(ctx, chi.RouteCtxKey, chi.NewRouteContext())
r = r.WithContext(ctx)

const serverName = "custom-server-name"
tracing.Middleware(provider, tracing.DefaultRoutePatterns, serverName)(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
rw.WriteHeader(http.StatusOK)
})).ServeHTTP(rw, r)

require.NoError(t, provider.ForceFlush(ctx))
spans := recorder.Ended()
require.NotEmpty(t, spans)

var found bool
for _, span := range spans {
for _, attr := range span.Attributes() {
if attr.Value.Emit() == serverName {
found = true
}
}
}
require.True(t, found, "configured server name should appear in the span attributes")
}

func Test_Middleware(t *testing.T) {
t.Parallel()

Expand Down Expand Up @@ -389,7 +549,7 @@ func Test_Middleware(t *testing.T) {
ctx = context.WithValue(ctx, chi.RouteCtxKey, chi.NewRouteContext())
r = r.WithContext(ctx)

tracing.Middleware(fake)(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
tracing.Middleware(fake, tracing.DefaultRoutePatterns, "coderd")(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
rw.WriteHeader(http.StatusNoContent)
})).ServeHTTP(rw, r)

Expand Down Expand Up @@ -422,7 +582,7 @@ func Test_Middleware(t *testing.T) {
ctx = context.WithValue(ctx, chi.RouteCtxKey, chi.NewRouteContext())
r = r.WithContext(ctx)

tracing.Middleware(provider)(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
tracing.Middleware(provider, tracing.DefaultRoutePatterns, "coderd")(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
rw.WriteHeader(http.StatusOK)
})).ServeHTTP(rw, r)

Expand Down
2 changes: 1 addition & 1 deletion coderd/workspaceapps/proxy.go
Original file line number Diff line number Diff line change
Expand Up @@ -703,7 +703,7 @@ func (s *Server) proxyWorkspaceApp(rw http.ResponseWriter, r *http.Request, appT
}

// end span so we don't get long lived trace data
tracing.EndHTTPSpan(r, http.StatusOK, trace.SpanFromContext(ctx))
tracing.EndHTTPSpan(r, http.StatusOK, trace.SpanFromContext(ctx), "coderd")

report := newStatsReportFromSignedToken(appToken)
defer func() {
Expand Down
2 changes: 1 addition & 1 deletion enterprise/wsproxy/wsproxy.go
Original file line number Diff line number Diff line change
Expand Up @@ -361,7 +361,7 @@ func New(ctx context.Context, opts *Options) (*Server, error) {
httpmw.WithProfilingLabels,
tracing.StatusWriterMiddleware,
opts.CookieConfig.Middleware,
tracing.Middleware(s.TracerProvider),
tracing.Middleware(s.TracerProvider, tracing.DefaultRoutePatterns, "coderd"),
httpmw.AttachRequestID,
httpmw.ExtractRealIP(s.Options.RealIPConfig),
loggermw.Logger(s.Logger, func(r *http.Request) string {
Expand Down
Loading