-
Notifications
You must be signed in to change notification settings - Fork 192
Expand file tree
/
Copy pathssh_command_framework.go
More file actions
165 lines (148 loc) · 5.87 KB
/
Copy pathssh_command_framework.go
File metadata and controls
165 lines (148 loc) · 5.87 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
package vmhelpers
import (
"context"
"errors"
"fmt"
"strings"
"time"
)
// Default retry count and backoff between SSH transport retries inside runSSHCommandWithFramework.
const (
defaultSSHTransportRetryAttempts = 3
defaultSSHTransportRetryInterval = 2 * time.Second
)
// errSSHTransport is a sentinel that wraps errors originating from the SSH
// transport layer (banner timeout, network unreachable, authentication failure)
// as opposed to errors returned by the remote command itself.
// Callers can use errors.Is(err, errSSHTransport) to distinguish transport
// failures from remote-command failures and apply different retry strategies.
var errSSHTransport = errors.New("SSH transport error")
// sshCommandRunOptions configures logging, transport retries, and backoff for runSSHCommandWithFramework.
type sshCommandRunOptions struct {
description string
transportRetryAttempts int
retryInterval time.Duration
}
// classifySSHStderrCategory returns a descriptive category for known SSH
// transport stderr patterns so that logging stays specific. If no known
// pattern matches, it returns "".
func classifySSHStderrCategory(stderr string) (category string, retryable bool) {
if isSSHAuthenticationFailure(stderr) {
return "authentication", false
}
if isSSHBannerTimeoutFailure(stderr) {
return "banner-timeout", true
}
if isSSHNetworkUnreachableFailure(stderr) {
return "network", true
}
lower := strings.ToLower(strings.TrimSpace(stderr))
switch {
case strings.Contains(lower, "websocket: close 1006"):
return "websocket-eof", true
case strings.Contains(lower, "unexpected eof"):
return "unexpected-eof", true
case strings.Contains(lower, "broken pipe"):
return "broken-pipe", true
case strings.Contains(lower, "closed by remote host"),
strings.Contains(lower, "connection closed by"):
return "remote-host-closed", true
case strings.Contains(lower, "internal error occurred: dialing vm"):
return "dialing-vm", true
case strings.Contains(lower, "connection reset by peer"):
return "connection-reset", true
default:
return "", true
}
}
// classifySSHFailure decides whether a failure is SSH transport-level (vs remote command) and if retrying helps.
func classifySSHFailure(stderr string, err error) (isSSH bool, retryable bool, category string) {
if err == nil {
return false, false, ""
}
if errors.Is(err, context.DeadlineExceeded) {
return true, true, "timeout"
}
// Check stderr for a known pattern first — gives us the most specific
// category for logging regardless of exit code.
cat, catRetryable := classifySSHStderrCategory(stderr)
// Use known stderr patterns for transport classification. Exit code 255
// alone is not sufficient, because remote commands can legitimately exit
// 255 and surface as deterministic command failures.
if cat != "" {
return true, catRetryable, cat
}
return false, false, ""
}
// sshTransportRetryInterval is the pause between retries in retryOnSSHTransport after transport errors.
const sshTransportRetryInterval = 10 * time.Second
// retryOnSSHTransport retries fn whenever it returns an errSSHTransport error.
// Non-transport errors and nil are returned immediately. The retry loop is
// bounded by ctx — callers should set an appropriate deadline/timeout.
func retryOnSSHTransport(ctx context.Context, logf func(string, ...any), desc string, fn func(ctx context.Context) error) error {
var lastErr error
for attempt := 1; ; attempt++ {
lastErr = fn(ctx)
if lastErr == nil || !errors.Is(lastErr, errSSHTransport) {
return lastErr
}
if logf != nil {
logf("%s: SSH transport issue (attempt %d), retrying in %s: %v",
desc, attempt, sshTransportRetryInterval, lastErr)
}
timer := time.NewTimer(sshTransportRetryInterval)
select {
case <-ctx.Done():
timer.Stop()
return fmt.Errorf("%s: context expired while retrying SSH transport error: %w (last: %v)",
desc, ctx.Err(), lastErr)
case <-timer.C:
}
}
}
// runSSHCommandWithFramework runs virt.SSH with transport classification and bounded retries.
// This is the transport-level retry loop: it retries SSH connectivity failures
// (banner timeout, network unreachable, websocket EOF, broken pipe, connection
// reset) but returns immediately when the remote command itself ran and exited
// non-zero — application-level retries are the caller's responsibility.
func runSSHCommandWithFramework(ctx context.Context, virt Virtctl, namespace, vm string, opts sshCommandRunOptions, command ...string) (stdout, stderr string, err error) {
attempts := opts.transportRetryAttempts
if attempts <= 0 {
attempts = defaultSSHTransportRetryAttempts
}
interval := opts.retryInterval
if interval <= 0 {
interval = defaultSSHTransportRetryInterval
}
description := strings.TrimSpace(opts.description)
if description == "" {
description = "ssh command"
}
for attempt := 1; attempt <= attempts; attempt++ {
stdout, stderr, err = virt.SSH(ctx, namespace, vm, command...)
if err == nil {
return stdout, stderr, nil
}
isSSH, retryable, category := classifySSHFailure(stderr, err)
if !isSSH {
return stdout, stderr, err
}
if !retryable {
return stdout, stderr, fmt.Errorf("%w: %s on %s/%s: terminal SSH %s failure: %w",
errSSHTransport, description, namespace, vm, category, err)
}
if attempt >= attempts {
return stdout, stderr, fmt.Errorf("%w: %s on %s/%s: retryable SSH %s failure persisted after %d attempt(s): %w",
errSSHTransport, description, namespace, vm, category, attempts, err)
}
virt.Logf("%s on %s/%s: retryable SSH %s condition (attempt %d/%d): %s",
description, namespace, vm, category, attempt, attempts, FormatGuestCommandOutputForError(stderr))
select {
case <-ctx.Done():
return stdout, stderr, fmt.Errorf("%w: %s on %s/%s: context done during SSH retry backoff: %w",
errSSHTransport, description, namespace, vm, ctx.Err())
case <-time.After(interval):
}
}
return stdout, stderr, err
}