Skip to content
Merged
97 changes: 97 additions & 0 deletions cli/flightrecorder.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
package cli

import (
"errors"
"os"
"path/filepath"
"sort"
"time"

"github.com/adrg/xdg"
"golang.org/x/xerrors"

"cdr.dev/slog/v3"
)

const (
// defaultCLIFlightRecorderSize is the default number of log entries below the
// current log level that a command keeps in memory and emits to stderr when
// it fails. It is intentionally small so a failure prints a few lines of
// context rather than flooding the terminal.
defaultCLIFlightRecorderSize = 10
// maxCLIFlightRecorderSize caps the configurable flight recorder size to bound
// memory use.
maxCLIFlightRecorderSize = 10000
// keepSessionLogFiles is the number of ssh session log files retained per log
// directory; older files are pruned when a new session starts.
keepSessionLogFiles = 50
)

// flightRecorder returns logger writing to sink at the current display level
// (Info by default, Debug under --verbose) with a flight recorder attached. The
// recorder keeps entries below the current level in memory and emits them when
// an error is logged, so the detail leading up to a failure is captured without
// logging it during normal operation. The recorder is always attached so it
// keeps working if the level changes; at Debug it simply has nothing to record.
func (r *RootCmd) flightRecorder(logger slog.Logger, sink slog.Sink, size int64) slog.Logger {
level := slog.LevelInfo
if r.verbose {
level = slog.LevelDebug
}
return logger.AppendSinks(sink).Leveled(level).FlightRecorder(int(clampFlightRecorderSize(size)))
}

// defaultSessionLogDir returns the default directory for ssh session logs.
// Following the XDG Base Directory spec, logs are state data, so they live under
// the state directory rather than the cache or data directory.
func defaultSessionLogDir() string {
return filepath.Join(xdg.StateHome, "coder", "logs")
}

// clampFlightRecorderSize bounds a requested size to [0, maxCLIFlightRecorderSize].
func clampFlightRecorderSize(size int64) int64 {
if size < 0 {
return 0
}
if size > maxCLIFlightRecorderSize {
return maxCLIFlightRecorderSize
}
return size
}

// pruneSessionLogs removes the oldest coder-*.log files in dir, keeping at most
// keep of the most recent. It is best effort: it continues past individual
// failures and returns them joined so the caller can surface them.
func pruneSessionLogs(dir string, keep int) error {
matches, err := filepath.Glob(filepath.Join(dir, "coder-*.log"))
if err != nil {
return xerrors.Errorf("glob session logs in %q: %w", dir, err)
}
type logFile struct {
path string
modTime time.Time
}
var errs []error
files := make([]logFile, 0, len(matches))
for _, path := range matches {
info, err := os.Stat(path)
if err != nil {
errs = append(errs, xerrors.Errorf("stat %q: %w", path, err))
continue
}
files = append(files, logFile{path: path, modTime: info.ModTime()})
}
if len(files) <= keep {
return errors.Join(errs...)
}
// Newest first, then remove everything past the keep threshold.
sort.Slice(files, func(i, j int) bool {
return files[i].modTime.After(files[j].modTime)
})
for _, f := range files[keep:] {
if err := os.Remove(f.path); err != nil {
errs = append(errs, xerrors.Errorf("remove %q: %w", f.path, err))
}
}
return errors.Join(errs...)
}
157 changes: 157 additions & 0 deletions cli/flightrecorder_internal_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,157 @@
package cli

import (
"bytes"
"context"
"fmt"
"os"
"path/filepath"
"runtime"
"strings"
"testing"
"time"

"github.com/stretchr/testify/require"

"cdr.dev/slog/v3"
"cdr.dev/slog/v3/sloggers/sloghuman"
)

func TestClampFlightRecorderSize(t *testing.T) {
t.Parallel()

cases := []struct {
in int64
want int64
}{
{in: -1, want: 0},
{in: 0, want: 0},
{in: 1000, want: 1000},
{in: maxCLIFlightRecorderSize, want: maxCLIFlightRecorderSize},
{in: maxCLIFlightRecorderSize + 1, want: maxCLIFlightRecorderSize},
}
for _, c := range cases {
require.Equal(t, c.want, clampFlightRecorderSize(c.in))
}
}

func TestPruneSessionLogs(t *testing.T) {
t.Parallel()

dir := t.TempDir()
base := time.Now()
const total = 55
for i := 0; i < total; i++ {
path := filepath.Join(dir, fmt.Sprintf("coder-ssh-%03d.log", i))
require.NoError(t, os.WriteFile(path, []byte("x"), 0o600))
// Older index => older modtime, so the highest indexes are newest.
modTime := base.Add(time.Duration(i) * time.Minute)
require.NoError(t, os.Chtimes(path, modTime, modTime))
}
// A non-matching file should never be pruned.
other := filepath.Join(dir, "unrelated.txt")
require.NoError(t, os.WriteFile(other, []byte("x"), 0o600))

pruneErr := pruneSessionLogs(dir, keepSessionLogFiles)
require.NoError(t, pruneErr)

remaining, err := filepath.Glob(filepath.Join(dir, "coder-*.log"))
require.NoError(t, err)
require.Len(t, remaining, keepSessionLogFiles)
// The oldest (lowest index) files were removed; the newest were kept.
require.NoFileExists(t, filepath.Join(dir, "coder-ssh-000.log"))
require.FileExists(t, filepath.Join(dir, fmt.Sprintf("coder-ssh-%03d.log", total-1)))
require.FileExists(t, other)
}

func TestPruneSessionLogs_ReturnsRemoveErrors(t *testing.T) {
t.Parallel()

if runtime.GOOS == "windows" {
t.Skip("a read-only directory does not prevent file removal on Windows, so os.Remove would not fail")
}

if os.Geteuid() == 0 {
t.Skip("root ignores directory permissions, so os.Remove would not fail")
}

dir := t.TempDir()
base := time.Now()
const total = 3
for i := 0; i < total; i++ {
path := filepath.Join(dir, fmt.Sprintf("coder-ssh-%03d.log", i))
require.NoError(t, os.WriteFile(path, []byte("x"), 0o600))
modTime := base.Add(time.Duration(i) * time.Minute)
require.NoError(t, os.Chtimes(path, modTime, modTime))
}

// Removing a file requires write permission on the containing directory, so a
// read-only dir forces os.Remove to fail. Restore perms before t.TempDir's
// cleanup (which runs after this LIFO-ordered cleanup) tries to remove it.
require.NoError(t, os.Chmod(dir, 0o500))
t.Cleanup(func() { _ = os.Chmod(dir, 0o700) })

err := pruneSessionLogs(dir, 1)
require.Error(t, err, "prune should return the remove failures")
}

func TestDefaultSessionLogDir(t *testing.T) {
t.Parallel()

require.True(t, strings.HasSuffix(
filepath.ToSlash(defaultSessionLogDir()), "coder/logs"),
"got %q", defaultSessionLogDir())
}

func TestFlightRecorder_RecordsUntilError(t *testing.T) {
t.Parallel()

var buf bytes.Buffer
r := &RootCmd{}
logger := r.flightRecorder(slog.Make(), sloghuman.Sink(&buf), 10)

ctx := context.Background()
logger.Debug(ctx, "recorded-debug")
logger.Sync()
require.NotContains(t, buf.String(), "recorded-debug",
"debug entry should be recorded, not written")

// Logging an error flushes the recorded history before the error entry.
logger.Error(ctx, "command failed for test")
logger.Sync()
got := buf.String()
require.Contains(t, got, "recorded-debug")
require.Contains(t, got, "command failed for test")
}

func TestFlightRecorder_FlushEmitsHistory(t *testing.T) {
t.Parallel()

// Flush is the path flightRecorderMiddleware uses to emit the recorded
// history on a returned error without logging a duplicate error line.
var buf bytes.Buffer
r := &RootCmd{}
logger := r.flightRecorder(slog.Make(), sloghuman.Sink(&buf), 10)

ctx := context.Background()
logger.Debug(ctx, "recorded-debug")
logger.Sync()
require.NotContains(t, buf.String(), "recorded-debug")

logger.Flush(ctx)
logger.Sync()
require.Contains(t, buf.String(), "recorded-debug")
}

func TestFlightRecorder_VerboseWritesDebug(t *testing.T) {
t.Parallel()

var buf bytes.Buffer
r := &RootCmd{verbose: true}
logger := r.flightRecorder(slog.Make(), sloghuman.Sink(&buf), 10)

logger.Debug(context.Background(), "verbose-debug")
logger.Sync()
require.Contains(t, buf.String(), "verbose-debug",
"verbose should write debug entries directly")
}
65 changes: 57 additions & 8 deletions cli/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (
"runtime"
"runtime/trace"
"slices"
"strconv"
"strings"
"sync"
"syscall"
Expand All @@ -32,6 +33,7 @@ import (
"golang.org/x/xerrors"

"cdr.dev/slog/v3"
"cdr.dev/slog/v3/sloggers/sloghuman"
"github.com/coder/coder/v2/buildinfo"
"github.com/coder/coder/v2/cli/cliui"
"github.com/coder/coder/v2/cli/config"
Expand Down Expand Up @@ -75,6 +77,7 @@ const (
varAllowRedirects = "allow-redirects"
varForceTty = "force-tty"
varVerbose = "verbose"
varFlightRecorderSize = "flight-recorder-size"
varDisableDirect = "disable-direct-connections"
varDisableNetworkTelemetry = "disable-network-telemetry"
varUseKeyring = "use-keyring"
Expand Down Expand Up @@ -405,9 +408,9 @@ func (r *RootCmd) Command(subcommands []*serpent.Command) (*serpent.Command, err
// the invocation context for every downstream middleware and handler.
cmd.Walk(func(cmd *serpent.Command) {
if cmd.Middleware == nil {
cmd.Middleware = serpent.Chain(clientSessionIDMiddleware(), PrintDeprecatedOptions())
cmd.Middleware = serpent.Chain(clientSessionIDMiddleware(), r.flightRecorderMiddleware(), PrintDeprecatedOptions())
} else {
cmd.Middleware = serpent.Chain(clientSessionIDMiddleware(), cmd.Middleware, PrintDeprecatedOptions())
cmd.Middleware = serpent.Chain(clientSessionIDMiddleware(), r.flightRecorderMiddleware(), cmd.Middleware, PrintDeprecatedOptions())
}
})

Expand Down Expand Up @@ -499,6 +502,15 @@ func (r *RootCmd) Command(subcommands []*serpent.Command) (*serpent.Command, err
Value: serpent.BoolOf(&r.verbose),
Group: globalGroup,
},
{
Flag: varFlightRecorderSize,
Env: "CODER_FLIGHT_RECORDER_SIZE",
Default: strconv.Itoa(defaultCLIFlightRecorderSize),
Description: "Number of log entries below the current log level to keep " +
"in memory and emit on errors. Set to 0 to disable the flight recorder.",
Value: serpent.Int64Of(&r.flightRecorderSize),
Group: globalGroup,
},
{
Flag: varDisableDirect,
Env: "CODER_DISABLE_DIRECT_CONNECTIONS",
Expand Down Expand Up @@ -584,12 +596,13 @@ type RootCmd struct {
header []string
headerCommand string

forceTTY bool
noOpen bool
verbose bool
versionFlag bool
disableDirect bool
debugHTTP bool
forceTTY bool
noOpen bool
verbose bool
flightRecorderSize int64
versionFlag bool
disableDirect bool
debugHTTP bool

disableNetworkTelemetry bool
noVersionCheck bool
Expand Down Expand Up @@ -1816,6 +1829,42 @@ const clientSessionIDEnv = "CODER_TRACE_SESSION_ID"
// carry a meaningless session ID in their logs, request baggage, or telemetry.
const annotationClientSessionID = "client_session_id"

// annotationFlightRecorder marks commands whose diagnostic logs should be kept
// in memory by a flight recorder and emitted to stderr only when the command
// returns an error. flightRecorderMiddleware installs the recorder for these
// commands. Commands that manage their own logger destination (for example ssh,
// which writes to a file to avoid corrupting its stdio stream) should not opt in.
const annotationFlightRecorder = "flight_recorder"

// flightRecorderMiddleware installs a stderr logger backed by a flight recorder
// for commands that opt in with annotationFlightRecorder. Entries below the
// display level (Info, or Debug under --verbose) are kept in a bounded in-memory
// ring and emitted only when the command returns an error, so successful runs
// stay quiet while the detail leading up to a failure is still available. The
// recorder is shared with any logger derived from the invocation logger (such as
// the codersdk client logger), so flushing here also emits their recorded
// entries.
func (r *RootCmd) flightRecorderMiddleware() serpent.MiddlewareFunc {
return func(next serpent.HandlerFunc) serpent.HandlerFunc {
return func(inv *serpent.Invocation) error {
if inv.IsCompletionMode() || inv.Command == nil ||
!inv.Command.Annotations.IsSet(annotationFlightRecorder) {
return next(inv)
}
logger := r.flightRecorder(inv.Logger, sloghuman.Sink(inv.Stderr), r.flightRecorderSize)
inv.Logger = logger
err := next(inv)
if err != nil {
// Replay the recorded diagnostic history to stderr. Flush does not
// add a duplicate error line; the returned error is rendered by the
// top-level formatter.
logger.Flush(inv.Context())
}
return err
}
}
}

type clientSessionIDContextKey struct{}

// withClientSessionID returns a copy of ctx carrying the client session ID so
Expand Down
Loading
Loading