Skip to content

Commit ea2fa20

Browse files
ctelferctelfer
committed
Add endpoint load-balancing mode
This is the heart of the scalability change for services in libnetwork. The present routing mesh adds load-balancing rules for a network to every container connected to the network. This newer approach creates a load-balancing endpoint per network per node. For every service on a network, libnetwork assigns the VIP of the service to the endpoint's interface as an alias. This endpoint must have a unique IP address in order to route return traffic to it. Traffic destined for a service's VIP arrives at the load-balancing endpoint on the VIP and from there, Linux load balances it among backend destinations while SNATing said traffic to the endpoint's unique IP address. The net result of this scheme is that each node in a swarm need only have one set of load balancing state per service instead of one per container on the node. This scheme is very similar to how services currently operate on Windows nodes in libnetwork. It (as with Windows nodes) costs the use of extra IP addresses in a network (one per node) and an extra network hop in the stack, although, always in the stack local to the container. In order to prevent existing deployments from suddenly failing if they failed to allocate sufficient address space to include per-node load-balancing endpoint IP addresses, this patch preserves the existing functionality and activates the new functionality on a per-network basis depending on whether the network has a load-balancing endpoint. Eventually, moby should always set this option when creating new networks and should only omit it for networks created as part of a swarm that are not marked to use endpoint load balancing. This patch also normalizes the code to treat "load" and "balancer" as two separate words from the perspectives of variable/function naming. This means that the 'b' in "balancer" must be capitalized. Signed-off-by: Chris Telfer <ctelfer@docker.com>
1 parent 85a3483 commit ea2fa20

File tree

8 files changed

+141
-176
lines changed

8 files changed

+141
-176
lines changed

‎libnetwork/controller.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -871,7 +871,7 @@ addToStore:
871871
}
872872
}()
873873

874-
if len(network.loadBalancerIP) != 0 {
874+
if network.hasLoadBalancerEndpoint() {
875875
if err = network.createLoadBalancerSandbox(); err != nil {
876876
return nil, err
877877
}

‎libnetwork/endpoint.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -540,6 +540,12 @@ func (ep *endpoint) sbJoin(sb *sandbox, options ...EndpointOption) (err error) {
540540
}
541541
}()
542542

543+
// Load balancing endpoints should never have a default gateway nor
544+
// should they alter the status of a network's default gateway
545+
if ep.loadBalancer && !sb.ingress {
546+
return nil
547+
}
548+
543549
if sb.needDefaultGW() && sb.getEndpointInGWNetwork() == nil {
544550
return sb.setupDefaultGW()
545551
}

‎libnetwork/network.go‎

Lines changed: 13 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -997,8 +997,8 @@ func (n *network) delete(force bool, rmLBEndpoint bool) error {
997997
}
998998

999999
// Check that the network is empty
1000-
var emptyCount uint64 = 0
1001-
if len(n.loadBalancerIP) != 0 {
1000+
var emptyCount uint64
1001+
if n.hasLoadBalancerEndpoint() {
10021002
emptyCount = 1
10031003
}
10041004
if !force && n.getEpCnt().EndpointCnt() > emptyCount {
@@ -1008,7 +1008,7 @@ func (n *network) delete(force bool, rmLBEndpoint bool) error {
10081008
return &ActiveEndpointsError{name: n.name, id: n.id}
10091009
}
10101010

1011-
if len(n.loadBalancerIP) != 0 {
1011+
if n.hasLoadBalancerEndpoint() {
10121012
// If we got to this point, then the following must hold:
10131013
// * force is true OR endpoint count == 1
10141014
if err := n.deleteLoadBalancerSandbox(); err != nil {
@@ -1077,9 +1077,6 @@ func (n *network) delete(force bool, rmLBEndpoint bool) error {
10771077
// Cleanup the service discovery for this network
10781078
c.cleanupServiceDiscovery(n.ID())
10791079

1080-
// Cleanup the load balancer
1081-
c.cleanupServiceBindings(n.ID())
1082-
10831080
removeFromStore:
10841081
// deleteFromStore performs an atomic delete operation and the
10851082
// network.epCnt will help prevent any possible
@@ -1931,6 +1928,10 @@ func (n *network) hasSpecialDriver() bool {
19311928
return n.Type() == "host" || n.Type() == "null"
19321929
}
19331930

1931+
func (n *network) hasLoadBalancerEndpoint() bool {
1932+
return len(n.loadBalancerIP) != 0
1933+
}
1934+
19341935
func (n *network) ResolveName(req string, ipType int) ([]net.IP, bool) {
19351936
var ipv6Miss bool
19361937

@@ -2111,9 +2112,9 @@ func (c *controller) getConfigNetwork(name string) (*network, error) {
21112112
}
21122113

21132114
func (n *network) lbSandboxName() string {
2114-
name := n.name + "-sbox"
2115+
name := "lb-" + n.name
21152116
if n.ingress {
2116-
name = "lb-" + n.name
2117+
name = n.name + "-sbox"
21172118
}
21182119
return name
21192120
}
@@ -2145,6 +2146,10 @@ func (n *network) createLoadBalancerSandbox() (retErr error) {
21452146
CreateOptionIpam(n.loadBalancerIP, nil, nil, nil),
21462147
CreateOptionLoadBalancer(),
21472148
}
2149+
if n.hasLoadBalancerEndpoint() && !n.ingress {
2150+
// Mark LB endpoints as anonymous so they don't show up in DNS
2151+
epOptions = append(epOptions, CreateOptionAnonymous())
2152+
}
21482153
ep, err := n.createEndpoint(endpointName, epOptions...)
21492154
if err != nil {
21502155
return err

‎libnetwork/sandbox.go‎

Lines changed: 1 addition & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -740,16 +740,8 @@ func releaseOSSboxResources(osSbox osl.Sandbox, ep *endpoint) {
740740

741741
ep.Lock()
742742
joinInfo := ep.joinInfo
743-
vip := ep.virtualIP
744743
ep.Unlock()
745744

746-
if len(vip) != 0 {
747-
loopName := osSbox.GetLoopbackIfaceName()
748-
if err := osSbox.RemoveAliasIP(loopName, &net.IPNet{IP: vip, Mask: net.CIDRMask(32, 32)}); err != nil {
749-
logrus.Warnf("Remove virtual IP %v failed: %v", vip, err)
750-
}
751-
}
752-
753745
if joinInfo == nil {
754746
return
755747
}
@@ -862,14 +854,6 @@ func (sb *sandbox) populateNetworkResources(ep *endpoint) error {
862854
}
863855
}
864856

865-
if len(ep.virtualIP) != 0 {
866-
loopName := sb.osSbox.GetLoopbackIfaceName()
867-
err := sb.osSbox.AddAliasIP(loopName, &net.IPNet{IP: ep.virtualIP, Mask: net.CIDRMask(32, 32)})
868-
if err != nil {
869-
return fmt.Errorf("failed to add virtual IP %v: %v", ep.virtualIP, err)
870-
}
871-
}
872-
873857
if joinInfo != nil {
874858
// Set up non-interface routes.
875859
for _, r := range joinInfo.StaticRoutes {
@@ -895,7 +879,7 @@ func (sb *sandbox) populateNetworkResources(ep *endpoint) error {
895879
// information including gateway and other routes so that
896880
// loadbalancers are populated all the network state is in
897881
// place in the sandbox.
898-
sb.populateLoadbalancers(ep)
882+
sb.populateLoadBalancers(ep)
899883

900884
// Only update the store if we did not come here as part of
901885
// sandbox delete. If we came here as part of delete then do

‎libnetwork/service_common.go‎

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -289,8 +289,7 @@ func (c *controller) addServiceBinding(svcName, svcID, nID, eID, containerName s
289289
logrus.Warnf("addServiceBinding %s possible transient state ok:%t entries:%d set:%t %s", eID, ok, entries, b, setStr)
290290
}
291291

292-
// Add loadbalancer service and backend in all sandboxes in
293-
// the network only if vip is valid.
292+
// Add loadbalancer service and backend to the network
294293
n.(*network).addLBBackend(ip, lb)
295294

296295
// Add the appropriate name resolutions
@@ -305,11 +304,6 @@ func (c *controller) rmServiceBinding(svcName, svcID, nID, eID, containerName st
305304

306305
var rmService bool
307306

308-
n, err := c.NetworkByID(nID)
309-
if err != nil {
310-
return err
311-
}
312-
313307
skey := serviceKey{
314308
id: svcID,
315309
ports: portConfigs(ingressPorts).String(),
@@ -367,7 +361,14 @@ func (c *controller) rmServiceBinding(svcName, svcID, nID, eID, containerName st
367361
// Remove loadbalancer service(if needed) and backend in all
368362
// sandboxes in the network only if the vip is valid.
369363
if entries == 0 {
370-
n.(*network).rmLBBackend(ip, lb, rmService, fullRemove)
364+
// The network may well have been deleted before the last
365+
// of the service bindings. That's ok, because removing
366+
// the network sandbox implicitly removes the backend
367+
// service bindings.
368+
n, err := c.NetworkByID(nID)
369+
if err == nil {
370+
n.(*network).rmLBBackend(ip, lb, rmService, fullRemove)
371+
}
371372
}
372373

373374
// Delete the name resolutions

0 commit comments

Comments
 (0)