Skip to content

Commit 055c5dd

Browse files
coolljt0725aboch
coolljt0725
authored and
aboch
committed
Add network restore to support docker live restore container
Signed-off-by: Lei Jitang <leijitang@huawei.com>
1 parent bbba96f commit 055c5dd

18 files changed

+392
-47
lines changed

‎libnetwork/config/config.go‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,10 @@ import (
1515

1616
// Config encapsulates configurations of various Libnetwork components
1717
type Config struct {
18-
Daemon DaemonCfg
19-
Cluster ClusterCfg
20-
Scopes map[string]*datastore.ScopeCfg
18+
Daemon DaemonCfg
19+
Cluster ClusterCfg
20+
Scopes map[string]*datastore.ScopeCfg
21+
ActiveSandboxes map[string]interface{}
2122
}
2223

2324
// DaemonCfg represents libnetwork core configuration
@@ -245,3 +246,11 @@ func OptionLocalKVProviderConfig(config *store.Config) Option {
245246
c.Scopes[datastore.LocalScope].Client.Config = config
246247
}
247248
}
249+
250+
// OptionActiveSandboxes function returns an option setter for passing the sandboxes
251+
// which were active during previous daemon life
252+
func OptionActiveSandboxes(sandboxes map[string]interface{}) Option {
253+
return func(c *Config) {
254+
c.ActiveSandboxes = sandboxes
255+
}
256+
}

‎libnetwork/controller.go‎

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -203,15 +203,13 @@ func New(cfgOptions ...config.Option) (NetworkController, error) {
203203
}
204204
}
205205

206-
// Reserve pools first before doing cleanup. This is because
207-
// if the pools are not populated properly, the cleanups of
208-
// endpoint/network and sandbox below will not be able to
209-
// release ip subnets and addresses properly into the pool
210-
// because the pools won't exist.
206+
// Reserve pools first before doing cleanup. Otherwise the
207+
// cleanups of endpoint/network and sandbox below will
208+
// generate many unnecessary warnings
211209
c.reservePools()
212210

213211
// Cleanup resources
214-
c.sandboxCleanup()
212+
c.sandboxCleanup(c.cfg.ActiveSandboxes)
215213
c.cleanupLocalEndpoints()
216214
c.networkCleanup()
217215

@@ -832,7 +830,7 @@ func (c *controller) NewSandbox(containerID string, options ...SandboxOption) (s
832830

833831
if sb.config.useDefaultSandBox {
834832
c.sboxOnce.Do(func() {
835-
c.defOsSbox, err = osl.NewSandbox(sb.Key(), false)
833+
c.defOsSbox, err = osl.NewSandbox(sb.Key(), false, false)
836834
})
837835

838836
if err != nil {
@@ -844,7 +842,7 @@ func (c *controller) NewSandbox(containerID string, options ...SandboxOption) (s
844842
}
845843

846844
if sb.osSbox == nil && !sb.config.useExternalKey {
847-
if sb.osSbox, err = osl.NewSandbox(sb.Key(), !sb.config.useDefaultSandBox); err != nil {
845+
if sb.osSbox, err = osl.NewSandbox(sb.Key(), !sb.config.useDefaultSandBox, false); err != nil {
848846
return nil, fmt.Errorf("failed to create new osl sandbox: %v", err)
849847
}
850848
}

‎libnetwork/drivers/bridge/bridge_store.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -184,7 +184,7 @@ func (ncfg *networkConfiguration) Exists() bool {
184184
}
185185

186186
func (ncfg *networkConfiguration) Skip() bool {
187-
return ncfg.DefaultBridge
187+
return false
188188
}
189189

190190
func (ncfg *networkConfiguration) New() datastore.KVObject {

‎libnetwork/drivers/overlay/ov_network.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -512,7 +512,7 @@ func (n *network) initSandbox() error {
512512
n.cleanupStaleSandboxes()
513513

514514
sbox, err := osl.NewSandbox(
515-
osl.GenerateKey(fmt.Sprintf("%d-", n.initEpoch)+n.id), !hostMode)
515+
osl.GenerateKey(fmt.Sprintf("%d-", n.initEpoch)+n.id), !hostMode, false)
516516
if err != nil {
517517
return fmt.Errorf("could not create network sandbox: %v", err)
518518
}

‎libnetwork/endpoint.go‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,7 @@ func (ep *endpoint) MarshalJSON() ([]byte, error) {
8484
epMap["name"] = ep.name
8585
epMap["id"] = ep.id
8686
epMap["ep_iface"] = ep.iface
87+
epMap["joinInfo"] = ep.joinInfo
8788
epMap["exposed_ports"] = ep.exposedPorts
8889
if ep.generic != nil {
8990
epMap["generic"] = ep.generic
@@ -115,6 +116,9 @@ func (ep *endpoint) UnmarshalJSON(b []byte) (err error) {
115116
ib, _ := json.Marshal(epMap["ep_iface"])
116117
json.Unmarshal(ib, &ep.iface)
117118

119+
jb, _ := json.Marshal(epMap["joinInfo"])
120+
json.Unmarshal(jb, &ep.joinInfo)
121+
118122
tb, _ := json.Marshal(epMap["exposed_ports"])
119123
var tPorts []types.TransportPort
120124
json.Unmarshal(tb, &tPorts)
@@ -235,6 +239,11 @@ func (ep *endpoint) CopyTo(o datastore.KVObject) error {
235239
ep.iface.CopyTo(dstEp.iface)
236240
}
237241

242+
if ep.joinInfo != nil {
243+
dstEp.joinInfo = &endpointJoinInfo{}
244+
ep.joinInfo.CopyTo(dstEp.joinInfo)
245+
}
246+
238247
dstEp.exposedPorts = make([]types.TransportPort, len(ep.exposedPorts))
239248
copy(dstEp.exposedPorts, ep.exposedPorts)
240249

@@ -1073,6 +1082,13 @@ func (ep *endpoint) releaseAddress() {
10731082
}
10741083

10751084
func (c *controller) cleanupLocalEndpoints() {
1085+
// Get used endpoints
1086+
eps := make(map[string]interface{})
1087+
for _, sb := range c.sandboxes {
1088+
for _, ep := range sb.endpoints {
1089+
eps[ep.id] = true
1090+
}
1091+
}
10761092
nl, err := c.getNetworksForScope(datastore.LocalScope)
10771093
if err != nil {
10781094
log.Warnf("Could not get list of networks during endpoint cleanup: %v", err)
@@ -1087,6 +1103,9 @@ func (c *controller) cleanupLocalEndpoints() {
10871103
}
10881104

10891105
for _, ep := range epl {
1106+
if _, ok := eps[ep.id]; ok {
1107+
continue
1108+
}
10901109
log.Infof("Removing stale endpoint %s (%s)", ep.name, ep.id)
10911110
if err := ep.Delete(true); err != nil {
10921111
log.Warnf("Could not delete local endpoint %s during endpoint cleanup: %v", ep.name, err)

‎libnetwork/endpoint_info.go‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -414,3 +414,56 @@ func (ep *endpoint) DisableGatewayService() {
414414

415415
ep.joinInfo.disableGatewayService = true
416416
}
417+
418+
func (epj *endpointJoinInfo) MarshalJSON() ([]byte, error) {
419+
epMap := make(map[string]interface{})
420+
if epj.gw != nil {
421+
epMap["gw"] = epj.gw.String()
422+
}
423+
if epj.gw6 != nil {
424+
epMap["gw6"] = epj.gw6.String()
425+
}
426+
epMap["disableGatewayService"] = epj.disableGatewayService
427+
epMap["StaticRoutes"] = epj.StaticRoutes
428+
return json.Marshal(epMap)
429+
}
430+
431+
func (epj *endpointJoinInfo) UnmarshalJSON(b []byte) error {
432+
var (
433+
err error
434+
epMap map[string]interface{}
435+
)
436+
if err = json.Unmarshal(b, &epMap); err != nil {
437+
return err
438+
}
439+
if v, ok := epMap["gw"]; ok {
440+
epj.gw6 = net.ParseIP(v.(string))
441+
}
442+
if v, ok := epMap["gw6"]; ok {
443+
epj.gw6 = net.ParseIP(v.(string))
444+
}
445+
epj.disableGatewayService = epMap["disableGatewayService"].(bool)
446+
447+
var tStaticRoute []types.StaticRoute
448+
if v, ok := epMap["StaticRoutes"]; ok {
449+
tb, _ := json.Marshal(v)
450+
var tStaticRoute []types.StaticRoute
451+
json.Unmarshal(tb, &tStaticRoute)
452+
}
453+
var StaticRoutes []*types.StaticRoute
454+
for _, r := range tStaticRoute {
455+
StaticRoutes = append(StaticRoutes, &r)
456+
}
457+
epj.StaticRoutes = StaticRoutes
458+
459+
return nil
460+
}
461+
462+
func (epj *endpointJoinInfo) CopyTo(dstEpj *endpointJoinInfo) error {
463+
dstEpj.disableGatewayService = epj.disableGatewayService
464+
dstEpj.StaticRoutes = make([]*types.StaticRoute, len(epj.StaticRoutes))
465+
copy(dstEpj.StaticRoutes, epj.StaticRoutes)
466+
dstEpj.gw = types.GetIPCopy(epj.gw)
467+
dstEpj.gw = types.GetIPCopy(epj.gw6)
468+
return nil
469+
}

‎libnetwork/libnetwork_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1305,7 +1305,7 @@ func externalKeyTest(t *testing.T, reexec bool) {
13051305
}
13061306

13071307
// Create a new OS sandbox using the osl API before using it in SetKey
1308-
if extOsBox, err := osl.NewSandbox("ValidKey", true); err != nil {
1308+
if extOsBox, err := osl.NewSandbox("ValidKey", true, false); err != nil {
13091309
t.Fatalf("Failed to create new osl sandbox")
13101310
} else {
13111311
defer func() {

‎libnetwork/osl/namespace_linux.go‎

Lines changed: 147 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,13 @@ package osl
22

33
import (
44
"fmt"
5+
"io/ioutil"
56
"net"
67
"os"
78
"os/exec"
89
"runtime"
10+
"strconv"
11+
"strings"
912
"sync"
1013
"syscall"
1114
"time"
@@ -133,6 +136,39 @@ func GC() {
133136
// container id.
134137
func GenerateKey(containerID string) string {
135138
maxLen := 12
139+
// Read sandbox key from host for overlay
140+
if strings.HasPrefix(containerID, "-") {
141+
var (
142+
index int
143+
indexStr string
144+
tmpkey string
145+
)
146+
dir, err := ioutil.ReadDir(prefix)
147+
if err != nil {
148+
return ""
149+
}
150+
151+
for _, v := range dir {
152+
id := v.Name()
153+
if strings.HasSuffix(id, containerID[:maxLen-1]) {
154+
indexStr = strings.TrimSuffix(id, containerID[:maxLen-1])
155+
tmpindex, err := strconv.Atoi(indexStr)
156+
if err != nil {
157+
return ""
158+
}
159+
if tmpindex > index {
160+
index = tmpindex
161+
tmpkey = id
162+
}
163+
164+
}
165+
}
166+
containerID = tmpkey
167+
if containerID == "" {
168+
return ""
169+
}
170+
}
171+
136172
if len(containerID) < maxLen {
137173
maxLen = len(containerID)
138174
}
@@ -142,10 +178,12 @@ func GenerateKey(containerID string) string {
142178

143179
// NewSandbox provides a new sandbox instance created in an os specific way
144180
// provided a key which uniquely identifies the sandbox
145-
func NewSandbox(key string, osCreate bool) (Sandbox, error) {
146-
err := createNetworkNamespace(key, osCreate)
147-
if err != nil {
148-
return nil, err
181+
func NewSandbox(key string, osCreate, isRestore bool) (Sandbox, error) {
182+
if !isRestore {
183+
err := createNetworkNamespace(key, osCreate)
184+
if err != nil {
185+
return nil, err
186+
}
149187
}
150188

151189
n := &networkNamespace{path: key, isDefault: !osCreate}
@@ -347,3 +385,108 @@ func (n *networkNamespace) Destroy() error {
347385
addToGarbagePaths(n.path)
348386
return nil
349387
}
388+
389+
// Restore restore the network namespace
390+
func (n *networkNamespace) Restore(ifsopt map[string][]IfaceOption, routes []*types.StaticRoute, gw net.IP, gw6 net.IP) error {
391+
// restore interfaces
392+
for name, opts := range ifsopt {
393+
if !strings.Contains(name, "+") {
394+
return fmt.Errorf("wrong iface name in restore osl sandbox interface: %s", name)
395+
}
396+
seps := strings.Split(name, "+")
397+
srcName := seps[0]
398+
dstPrefix := seps[1]
399+
i := &nwIface{srcName: srcName, dstName: dstPrefix, ns: n}
400+
i.processInterfaceOptions(opts...)
401+
if i.master != "" {
402+
i.dstMaster = n.findDst(i.master, true)
403+
if i.dstMaster == "" {
404+
return fmt.Errorf("could not find an appropriate master %q for %q",
405+
i.master, i.srcName)
406+
}
407+
}
408+
if n.isDefault {
409+
i.dstName = i.srcName
410+
} else {
411+
// due to the docker network connect/disconnect, so the dstName should
412+
// restore from the namespace
413+
err := nsInvoke(n.path, func(nsFD int) error { return nil }, func(callerFD int) error {
414+
ifaces, err := net.Interfaces()
415+
if err != nil {
416+
return err
417+
}
418+
for _, iface := range ifaces {
419+
addrs, err := iface.Addrs()
420+
if err != nil {
421+
return err
422+
}
423+
if strings.HasPrefix(iface.Name, "vxlan") {
424+
if i.dstName == "vxlan" {
425+
i.dstName = iface.Name
426+
break
427+
}
428+
}
429+
// find the interface name by ip
430+
if i.address != nil {
431+
for _, addr := range addrs {
432+
if addr.String() == i.address.String() {
433+
i.dstName = iface.Name
434+
break
435+
}
436+
continue
437+
}
438+
if i.dstName == iface.Name {
439+
break
440+
}
441+
}
442+
// This is to find the interface name of the pair in overlay sandbox
443+
if strings.HasPrefix(iface.Name, "veth") {
444+
if i.master != "" && i.dstName == "veth" {
445+
i.dstName = iface.Name
446+
}
447+
}
448+
}
449+
return nil
450+
})
451+
if err != nil {
452+
return err
453+
}
454+
var index int
455+
indexStr := strings.TrimPrefix(i.dstName, dstPrefix)
456+
if indexStr != "" {
457+
index, err = strconv.Atoi(indexStr)
458+
if err != nil {
459+
return err
460+
}
461+
}
462+
index++
463+
n.Lock()
464+
if index > n.nextIfIndex {
465+
n.nextIfIndex = index
466+
}
467+
n.iFaces = append(n.iFaces, i)
468+
n.Unlock()
469+
}
470+
}
471+
472+
// restore routes
473+
for _, r := range routes {
474+
n.Lock()
475+
n.staticRoutes = append(n.staticRoutes, r)
476+
n.Unlock()
477+
}
478+
479+
// restore gateway
480+
if len(gw) > 0 {
481+
n.Lock()
482+
n.gw = gw
483+
n.Unlock()
484+
}
485+
486+
if len(gw6) > 0 {
487+
n.Lock()
488+
n.gwv6 = gw6
489+
n.Unlock()
490+
}
491+
return nil
492+
}

‎libnetwork/osl/namespace_windows.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ func GenerateKey(containerID string) string {
1515

1616
// NewSandbox provides a new sandbox instance created in an os specific way
1717
// provided a key which uniquely identifies the sandbox
18-
func NewSandbox(key string, osCreate bool) (Sandbox, error) {
18+
func NewSandbox(key string, osCreate, isRestore bool) (Sandbox, error) {
1919
return nil, nil
2020
}
2121

0 commit comments

Comments
 (0)