Skip to content

Commit c4cd238

Browse files
ehazlettehazlett
committed
Merge pull request docker-archive-public#1685 from nathanleclaire/daemon_wait_over_ssh
Fix Docker daemon wait
2 parents d063fe5 + 9453df4 commit c4cd238

File tree

11 files changed

+151
-89
lines changed

11 files changed

+151
-89
lines changed

‎drivers/virtualbox/virtualbox.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -136,7 +136,7 @@ func GetCreateFlags() []cli.Flag {
136136
}
137137

138138
func (d *Driver) GetSSHHostname() (string, error) {
139-
return "localhost", nil
139+
return "127.0.0.1", nil
140140
}
141141

142142
func (d *Driver) GetSSHUsername() string {

‎libmachine/drivers/utils.go‎

Lines changed: 9 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -2,17 +2,12 @@ package drivers
22

33
import (
44
"fmt"
5-
"strings"
65

76
"github.com/docker/machine/libmachine/log"
87
"github.com/docker/machine/libmachine/mcnutils"
98
"github.com/docker/machine/libmachine/ssh"
109
)
1110

12-
const (
13-
ErrExitCode255 = "255"
14-
)
15-
1611
func GetSSHClientFromDriver(d Driver) (ssh.Client, error) {
1712
addr, err := d.GetSSHHostname()
1813
if err != nil {
@@ -33,10 +28,6 @@ func GetSSHClientFromDriver(d Driver) (ssh.Client, error) {
3328

3429
}
3530

36-
func isErr255Exit(err error) bool {
37-
return strings.Contains(err.Error(), ErrExitCode255)
38-
}
39-
4031
func RunSSHCommandFromDriver(d Driver, command string) (string, error) {
4132
client, err := GetSSHClientFromDriver(d)
4233
if err != nil {
@@ -47,34 +38,21 @@ func RunSSHCommandFromDriver(d Driver, command string) (string, error) {
4738

4839
output, err := client.Output(command)
4940
log.Debugf("SSH cmd err, output: %v: %s", err, output)
50-
if err != nil && !isErr255Exit(err) {
51-
log.Error("SSH cmd error!")
52-
log.Errorf("command: %s", command)
53-
log.Errorf("err : %v", err)
54-
log.Fatalf("output : %s", output)
41+
if err != nil {
42+
returnedErr := fmt.Errorf(`Something went wrong running an SSH command!
43+
command : %s
44+
err : %v
45+
output : %s
46+
`, command, err, output)
47+
return "", returnedErr
5548
}
5649

57-
return output, err
50+
return output, nil
5851
}
5952

6053
func sshAvailableFunc(d Driver) func() bool {
6154
return func() bool {
6255
log.Debug("Getting to WaitForSSH function...")
63-
hostname, err := d.GetSSHHostname()
64-
if err != nil {
65-
log.Debugf("Error getting IP address waiting for SSH: %s", err)
66-
return false
67-
}
68-
port, err := d.GetSSHPort()
69-
if err != nil {
70-
log.Debugf("Error getting SSH port: %s", err)
71-
return false
72-
}
73-
if err := ssh.WaitForTCP(fmt.Sprintf("%s:%d", hostname, port)); err != nil {
74-
log.Debugf("Error waiting for TCP waiting for SSH: %s", err)
75-
return false
76-
}
77-
7856
if _, err := RunSSHCommandFromDriver(d, "exit 0"); err != nil {
7957
log.Debugf("Error getting ssh command 'exit 0' : %s", err)
8058
return false
@@ -85,7 +63,7 @@ func sshAvailableFunc(d Driver) func() bool {
8563

8664
func WaitForSSH(d Driver) error {
8765
if err := mcnutils.WaitFor(sshAvailableFunc(d)); err != nil {
88-
return fmt.Errorf("Too many retries. Last error: %s", err)
66+
return fmt.Errorf("Too many retries waiting for SSH to be available. Last error: %s", err)
8967
}
9068
return nil
9169
}

‎libmachine/host/host.go‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -175,7 +175,3 @@ func (h *Host) ConfigureAuth() error {
175175

176176
return nil
177177
}
178-
179-
func WaitForSSH(h *Host) error {
180-
return drivers.WaitForSSH(h.Driver)
181-
}

‎libmachine/libmachine.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ func Create(store persist.Store, h *host.Host) error {
5959
}
6060

6161
log.Info("Machine is running, waiting for SSH to be available...")
62-
if err := host.WaitForSSH(h); err != nil {
62+
if err := drivers.WaitForSSH(h.Driver); err != nil {
6363
return fmt.Errorf("Error waiting for SSH: %s", err)
6464
}
6565

‎libmachine/mcnutils/utils.go‎

Lines changed: 0 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@ import (
66
"encoding/json"
77
"fmt"
88
"io"
9-
"net"
109
"os"
1110
"runtime"
1211
"strconv"
@@ -95,18 +94,6 @@ func WaitFor(f func() bool) error {
9594
return WaitForSpecific(f, 60, 3*time.Second)
9695
}
9796

98-
func WaitForDocker(ip string, daemonPort int) error {
99-
return WaitFor(func() bool {
100-
conn, err := net.Dial("tcp", fmt.Sprintf("%s:%d", ip, daemonPort))
101-
if err != nil {
102-
log.Debugf("Daemon not responding yet: %s", err)
103-
return false
104-
}
105-
conn.Close()
106-
return true
107-
})
108-
}
109-
11097
func DumpVal(vals ...interface{}) {
11198
for _, val := range vals {
11299
prettyJSON, err := json.MarshalIndent(val, "", " ")

‎libmachine/provision/boot2docker.go‎

Lines changed: 32 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,10 @@ package provision
33
import (
44
"bytes"
55
"fmt"
6+
"net"
67
"path"
78
"text/template"
9+
"time"
810

911
"github.com/docker/machine/commands/mcndirs"
1012
"github.com/docker/machine/libmachine/auth"
@@ -182,6 +184,35 @@ func (provisioner *Boot2DockerProvisioner) GetOsReleaseInfo() (*OsRelease, error
182184
}
183185

184186
func (provisioner *Boot2DockerProvisioner) Provision(swarmOptions swarm.SwarmOptions, authOptions auth.AuthOptions, engineOptions engine.EngineOptions) error {
187+
const (
188+
dockerPort = 2376
189+
)
190+
191+
defer func() {
192+
ip, err := provisioner.Driver.GetIP()
193+
if err != nil {
194+
log.Fatalf("Could not get IP address for created machine: %s", err)
195+
}
196+
197+
if conn, err := net.DialTimeout("tcp", fmt.Sprintf("%s:%d", ip, dockerPort), 5*time.Second); err != nil {
198+
log.Warn(`
199+
This machine has been allocated an IP address, but Docker Machine could not
200+
reach it successfully.
201+
202+
SSH for the machine should still work, but connecting to exposed ports, such as
203+
the Docker daemon port (usually <ip>:2376), may not work properly.
204+
205+
You may need to add the route manually, or use another related workaround.
206+
207+
This could be due to a VPN, proxy, or host file configuration issue.
208+
209+
You also might want to clear any VirtualBox host only interfaces you are not using.`)
210+
log.Fatal(err)
211+
} else {
212+
conn.Close()
213+
}
214+
}()
215+
185216
provisioner.SwarmOptions = swarmOptions
186217
provisioner.AuthOptions = authOptions
187218
provisioner.EngineOptions = engineOptions
@@ -194,14 +225,9 @@ func (provisioner *Boot2DockerProvisioner) Provision(swarmOptions swarm.SwarmOpt
194225
return err
195226
}
196227

197-
ip, err := provisioner.GetDriver().GetIP()
198-
if err != nil {
199-
return err
200-
}
201-
202228
// b2d hosts need to wait for the daemon to be up
203229
// before continuing with provisioning
204-
if err := mcnutils.WaitForDocker(ip, 2376); err != nil {
230+
if err := waitForDocker(provisioner, dockerPort); err != nil {
205231
return err
206232
}
207233

‎libmachine/provision/coreos.go‎

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@ import (
99
"github.com/docker/machine/libmachine/drivers"
1010
"github.com/docker/machine/libmachine/engine"
1111
"github.com/docker/machine/libmachine/log"
12-
"github.com/docker/machine/libmachine/mcnutils"
1312
"github.com/docker/machine/libmachine/provision/pkgaction"
1413
"github.com/docker/machine/libmachine/provision/serviceaction"
1514
"github.com/docker/machine/libmachine/swarm"
@@ -59,11 +58,7 @@ func (provisioner *CoreOSProvisioner) Service(name string, action serviceaction.
5958

6059
// wait until docker is running
6160
if (name == "docker") && (action.String() == "start") {
62-
ip, err := provisioner.GetDriver().GetIP()
63-
if err != nil {
64-
return err
65-
}
66-
if err := mcnutils.WaitForDocker(ip, 2376); err != nil {
61+
if err := waitForDocker(provisioner, 2376); err != nil {
6762
return err
6863
}
6964
}

‎libmachine/provision/errors.go‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,25 @@ package provision
22

33
import (
44
"errors"
5+
"fmt"
56
)
67

78
var (
89
ErrDetectionFailed = errors.New("OS type not recognized")
910
ErrSSHCommandFailed = errors.New("SSH command failure")
1011
ErrNotImplemented = errors.New("Runtime not implemented")
1112
)
13+
14+
type ErrDaemonAvailable struct {
15+
wrappedErr error
16+
}
17+
18+
func (e ErrDaemonAvailable) Error() string {
19+
return fmt.Sprintf("Unable to verify the Docker daemon is listening: %s", e.wrappedErr)
20+
}
21+
22+
func NewErrDaemonAvailable(err error) ErrDaemonAvailable {
23+
return ErrDaemonAvailable{
24+
wrappedErr: err,
25+
}
26+
}

‎libmachine/provision/utils.go‎

Lines changed: 48 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,10 @@ import (
66
"net/url"
77
"path"
88
"path/filepath"
9+
"regexp"
910
"strconv"
1011
"strings"
12+
"time"
1113

1214
"github.com/docker/machine/libmachine/auth"
1315
"github.com/docker/machine/libmachine/cert"
@@ -58,12 +60,13 @@ func ConfigureAuth(p Provisioner) error {
5860
err error
5961
)
6062

61-
machineName := p.GetDriver().GetMachineName()
63+
driver := p.GetDriver()
64+
machineName := driver.GetMachineName()
6265
authOptions := p.GetAuthOptions()
6366
org := machineName
6467
bits := 2048
6568

66-
ip, err := p.GetDriver().GetIP()
69+
ip, err := driver.GetIP()
6770
if err != nil {
6871
return err
6972
}
@@ -143,7 +146,7 @@ func ConfigureAuth(p Provisioner) error {
143146
return err
144147
}
145148

146-
dockerUrl, err := p.GetDriver().GetURL()
149+
dockerUrl, err := driver.GetURL()
147150
if err != nil {
148151
return err
149152
}
@@ -176,10 +179,50 @@ func ConfigureAuth(p Provisioner) error {
176179
return err
177180
}
178181

179-
// TODO: Do not hardcode daemon port, ask the driver
180-
if err := mcnutils.WaitForDocker(ip, dockerPort); err != nil {
182+
if err := waitForDocker(p, dockerPort); err != nil {
181183
return err
182184
}
183185

184186
return nil
185187
}
188+
189+
func matchNetstatOut(reDaemonListening, netstatOut string) bool {
190+
// TODO: I would really prefer this be a Scanner directly on
191+
// the STDOUT of the executed command than to do all the string
192+
// manipulation hokey-pokey.
193+
//
194+
// TODO: Unit test this matching.
195+
for _, line := range strings.Split(netstatOut, "\n") {
196+
match, err := regexp.MatchString(reDaemonListening, line)
197+
if err != nil {
198+
log.Warnf("Regex warning: %s", err)
199+
}
200+
if match && line != "" {
201+
return true
202+
}
203+
}
204+
205+
return false
206+
}
207+
208+
func checkDaemonUp(p Provisioner, dockerPort int) func() bool {
209+
reDaemonListening := fmt.Sprintf(":%d.*LISTEN", dockerPort)
210+
return func() bool {
211+
// HACK: Check netstat's output to see if anyone's listening on the Docker API port.
212+
netstatOut, err := p.SSHCommand("netstat -a")
213+
if err != nil {
214+
log.Warnf("Error running SSH command: %s", err)
215+
return false
216+
}
217+
218+
return matchNetstatOut(reDaemonListening, netstatOut)
219+
}
220+
}
221+
222+
func waitForDocker(p Provisioner, dockerPort int) error {
223+
if err := mcnutils.WaitForSpecific(checkDaemonUp(p, dockerPort), 5, 3*time.Second); err != nil {
224+
return NewErrDaemonAvailable(err)
225+
}
226+
227+
return nil
228+
}

‎libmachine/provision/utils_test.go‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,50 @@ import (
1010
"github.com/docker/machine/libmachine/auth"
1111
)
1212

13+
var (
14+
reDaemonListening = ":2376.*LISTEN"
15+
)
16+
17+
func TestMatchNetstatOutMissing(t *testing.T) {
18+
nsOut := `Active Internet connections (servers and established)
19+
Proto Recv-Q Send-Q Local Address Foreign Address State
20+
tcp 0 0 0.0.0.0:ssh 0.0.0.0:* LISTEN
21+
tcp 0 72 192.168.25.141:ssh 192.168.25.1:63213 ESTABLISHED
22+
tcp 0 0 :::ssh :::* LISTEN
23+
Active UNIX domain sockets (servers and established)
24+
Proto RefCnt Flags Type State I-Node Path
25+
unix 2 [ ACC ] STREAM LISTENING 17990 /var/run/acpid.socket
26+
unix 2 [ ACC ] SEQPACKET LISTENING 14233 /run/udev/control
27+
unix 3 [ ] STREAM CONNECTED 18688
28+
unix 3 [ ] DGRAM 14243
29+
unix 3 [ ] STREAM CONNECTED 18689
30+
unix 3 [ ] DGRAM 14242`
31+
if matchNetstatOut(reDaemonListening, nsOut) {
32+
t.Fatal("Expected not to match the netstat output as showing the daemon listening but got a match")
33+
}
34+
}
35+
36+
func TestMatchNetstatOutPresent(t *testing.T) {
37+
nsOut := `Active Internet connections (servers and established)
38+
Proto Recv-Q Send-Q Local Address Foreign Address State
39+
tcp 0 0 0.0.0.0:ssh 0.0.0.0:* LISTEN
40+
tcp 0 72 192.168.25.141:ssh 192.168.25.1:63235 ESTABLISHED
41+
tcp 0 0 :::2376 :::* LISTEN
42+
tcp 0 0 :::ssh :::* LISTEN
43+
Active UNIX domain sockets (servers and established)
44+
Proto RefCnt Flags Type State I-Node Path
45+
unix 2 [ ACC ] STREAM LISTENING 17990 /var/run/acpid.socket
46+
unix 2 [ ACC ] SEQPACKET LISTENING 14233 /run/udev/control
47+
unix 2 [ ACC ] STREAM LISTENING 19365 /var/run/docker.sock
48+
unix 3 [ ] STREAM CONNECTED 19774
49+
unix 3 [ ] STREAM CONNECTED 19775
50+
unix 3 [ ] DGRAM 14243
51+
unix 3 [ ] DGRAM 14242`
52+
if !matchNetstatOut(reDaemonListening, nsOut) {
53+
t.Fatal("Expected to match the netstat output as showing the daemon listening but didn't")
54+
}
55+
}
56+
1357
func TestGenerateDockerOptionsBoot2Docker(t *testing.T) {
1458
p := &Boot2DockerProvisioner{
1559
Driver: &fakedriver.FakeDriver{},

0 commit comments

Comments
 (0)