Merge pull request moby#49150 from robmry/live_restore_fixes

Fix live restore for IPv6-only and multiple gateway endpoints

integration/network/network_linux_test.go

@@ -389,10 +389,17 @@ func checkCtrRoutes(t *testing.T, ctx context.Context, apiClient client.APIClien
 	})
 
 	assert.Check(t, is.Equal(len(routes), expRoutes), "expected %d routes, got %d:\n%s", expRoutes, len(routes), strings.Join(routes, "\n"))
-	defFound := slices.ContainsFunc(routes, func(s string) bool {
-		return strings.Contains(s, expDefRoute)
-	})
-	assert.Check(t, defFound, "default route %q not found:\n%s", expDefRoute, strings.Join(routes, "\n"))
+	if expDefRoute == "" {
+		defFound := slices.ContainsFunc(routes, func(s string) bool {
+			return strings.HasPrefix(s, "default")
+		})
+		assert.Check(t, !defFound, "unexpected default route\n%s", strings.Join(routes, "\n"))
+	} else {
+		defFound := slices.ContainsFunc(routes, func(s string) bool {
+			return strings.Contains(s, expDefRoute)
+		})
+		assert.Check(t, defFound, "default route %q not found:\n%s", expDefRoute, strings.Join(routes, "\n"))
+	}
 }
 
 // TestMixL3IPVlanAndBridge checks that a container can be connected to a layer-3
@@ -410,61 +417,110 @@ func TestMixL3IPVlanAndBridge(t *testing.T) {
 	skip.If(t, testEnv.IsRootless, "can't see the dummy parent interface from the rootless namespace")
 
 	ctx := testutil.StartSpan(baseContext, t)
-	d := daemon.New(t)
-	d.StartWithBusybox(ctx, t)
-	defer d.Stop(t)
-	c := d.NewClientT(t)
-	defer c.Close()
 
-	const brNetName = "brnet"
-	network.CreateNoError(ctx, t, c, brNetName,
-		network.WithOption(netlabel.ContainerIfacePrefix, "bri"),
-		network.WithIPv6(),
-		network.WithIPAM("192.168.123.0/24", "192.168.123.1"),
-		network.WithIPAM("fd6f:36f8:3005::/64", "fd6f:36f8:3005::1"),
-	)
-	defer network.RemoveNoError(ctx, t, c, brNetName)
-
-	// Create a dummy parent interface rather than letting the driver do it because,
-	// when the driver creates its own, it becomes a '--internal' network and no
-	// default route is configured.
-	const parentIfName = "di-dummy0"
-	CreateMasterDummy(ctx, t, parentIfName)
-	defer DeleteInterface(ctx, t, parentIfName)
-
-	const ipvNetName = "ipvnet"
-	network.CreateNoError(ctx, t, c, ipvNetName,
-		network.WithDriver("ipvlan"),
-		network.WithOption("ipvlan_mode", "l3"),
-		network.WithOption("parent", parentIfName),
-		network.WithIPv6(),
-		network.WithIPAM("192.168.124.0/24", ""),
-		network.WithIPAM("fd7d:8755:51ba::/64", ""),
-	)
-	defer network.RemoveNoError(ctx, t, c, ipvNetName)
-
-	// Create a container connected to both networks, bridge network acting as gateway.
-	ctrId := container.Run(ctx, t, c,
-		container.WithNetworkMode(brNetName),
-		container.WithEndpointSettings(brNetName,
-			&networktypes.EndpointSettings{GwPriority: 1},
-		),
-		container.WithEndpointSettings(ipvNetName, &networktypes.EndpointSettings{}),
-	)
-	defer container.Remove(ctx, t, c, ctrId, containertypes.RemoveOptions{Force: true})
-	// Expect three IPv4 routes: the default, plus one per network.
-	checkCtrRoutes(t, ctx, c, ctrId, syscall.AF_INET, 3, "default via 192.168.123.1 dev bri")
-	// Expect seven IPv6 routes: the default, plus UL, LL, and multicast routes per network.
-	checkCtrRoutes(t, ctx, c, ctrId, syscall.AF_INET6, 7, "default via fd6f:36f8:3005::1 dev bri")
-
-	// Disconnect the bridge network, expect the ipvlan's default route to be set up.
-	c.NetworkDisconnect(ctx, brNetName, ctrId, false)
-	checkCtrRoutes(t, ctx, c, ctrId, syscall.AF_INET, 2, "default dev eth")
-	checkCtrRoutes(t, ctx, c, ctrId, syscall.AF_INET6, 4, "default dev eth")
-
-	// Reconnect the bridge network, with gw-priority=1 so it gets the gateway back.
-	// Expect the ipvlan's default route to be removed.
-	c.NetworkConnect(ctx, brNetName, ctrId, &networktypes.EndpointSettings{GwPriority: 1})
-	checkCtrRoutes(t, ctx, c, ctrId, syscall.AF_INET, 3, "default via 192.168.123.1 dev bri")
-	checkCtrRoutes(t, ctx, c, ctrId, syscall.AF_INET6, 7, "default via fd6f:36f8:3005::1 dev bri")
+	tests := []struct {
+		name        string
+		liveRestore bool
+	}{
+		{
+			name: "no live restore",
+		},
+		{
+			// If the daemon is restarted with a running container, the osSbox structure
+			// must be repopulated correctly in order for gateways to be removed then
+			// re-added when network connections change.
+			name:        "live restore",
+			liveRestore: true,
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			ctx := testutil.StartSpan(ctx, t)
+
+			// experimental is needed for a WithIPv4(false) network.
+			d := daemon.New(t, daemon.WithExperimental())
+			var daemonArgs []string
+			if tc.liveRestore {
+				daemonArgs = append(daemonArgs, "--live-restore")
+			}
+			d.StartWithBusybox(ctx, t, daemonArgs...)
+			defer d.Stop(t)
+			c := d.NewClientT(t)
+			defer c.Close()
+
+			const br46NetName = "br46net"
+			network.CreateNoError(ctx, t, c, br46NetName,
+				network.WithOption(netlabel.ContainerIfacePrefix, "bds"),
+				network.WithIPv6(),
+				network.WithIPAM("192.168.123.0/24", "192.168.123.1"),
+				network.WithIPAM("fd6f:36f8:3005::/64", "fd6f:36f8:3005::1"),
+			)
+			defer network.RemoveNoError(ctx, t, c, br46NetName)
+
+			const br6NetName = "br6net"
+			network.CreateNoError(ctx, t, c, br6NetName,
+				network.WithOption(netlabel.ContainerIfacePrefix, "bss"),
+				network.WithIPv4(false),
+				network.WithIPv6(),
+				network.WithIPAM("fdc9:adaf:b5da::/64", "fdc9:adaf:b5da::1"),
+			)
+			defer network.RemoveNoError(ctx, t, c, br6NetName)
+
+			// Create a dummy parent interface rather than letting the driver do it because,
+			// when the driver creates its own, it becomes a '--internal' network and no
+			// default route is configured.
+			const parentIfName = "di-dummy0"
+			CreateMasterDummy(ctx, t, parentIfName)
+			defer DeleteInterface(ctx, t, parentIfName)
+
+			const ipvNetName = "ipvnet"
+			network.CreateNoError(ctx, t, c, ipvNetName,
+				network.WithDriver("ipvlan"),
+				network.WithOption("ipvlan_mode", "l3"),
+				network.WithOption("parent", parentIfName),
+				network.WithIPv6(),
+				network.WithIPAM("192.168.124.0/24", ""),
+				network.WithIPAM("fd7d:8755:51ba::/64", ""),
+			)
+			defer network.RemoveNoError(ctx, t, c, ipvNetName)
+
+			// Create a container connected to all three networks, bridge network acting as gateway.
+			ctrId := container.Run(ctx, t, c,
+				container.WithNetworkMode(br46NetName),
+				container.WithEndpointSettings(br46NetName,
+					&networktypes.EndpointSettings{GwPriority: 1},
+				),
+				container.WithEndpointSettings(br6NetName, &networktypes.EndpointSettings{}),
+				container.WithEndpointSettings(ipvNetName, &networktypes.EndpointSettings{}),
+			)
+			defer container.Remove(ctx, t, c, ctrId, containertypes.RemoveOptions{Force: true})
+
+			if tc.liveRestore {
+				d.Restart(t, daemonArgs...)
+			}
+
+			// Expect three IPv4 routes: the default, plus one per network.
+			checkCtrRoutes(t, ctx, c, ctrId, syscall.AF_INET, 3, "default via 192.168.123.1 dev bds")
+			// Expect ten IPv6 routes: the default, plus UL, LL, and multicast routes per network.
+			checkCtrRoutes(t, ctx, c, ctrId, syscall.AF_INET6, 10, "default via fd6f:36f8:3005::1 dev bds")
+
+			// Disconnect the dual-stack bridge network, expect the ipvlan's default route to be set up.
+			c.NetworkDisconnect(ctx, br46NetName, ctrId, false)
+			checkCtrRoutes(t, ctx, c, ctrId, syscall.AF_INET, 2, "default dev eth")
+			checkCtrRoutes(t, ctx, c, ctrId, syscall.AF_INET6, 7, "default dev eth")
+
+			// Disconnect the ipvlan, expect the IPv6-only network to be the gateway, with no IPv4 gateway.
+			// (For this to work in the live-restore case the "dstName" of the interface must have been
+			// restored in the osSbox, based on matching the running interface's IPv6 address.)
+			c.NetworkDisconnect(ctx, ipvNetName, ctrId, false)
+			checkCtrRoutes(t, ctx, c, ctrId, syscall.AF_INET, 0, "")
+			checkCtrRoutes(t, ctx, c, ctrId, syscall.AF_INET6, 4, "default via fdc9:adaf:b5da::1 dev bss")
+
+			// Reconnect the dual-stack bridge, expect it to be the gateway for both addr families.
+			c.NetworkConnect(ctx, br46NetName, ctrId, &networktypes.EndpointSettings{GwPriority: 1})
+			checkCtrRoutes(t, ctx, c, ctrId, syscall.AF_INET, 2, "default via 192.168.123.1 dev bds")
+			checkCtrRoutes(t, ctx, c, ctrId, syscall.AF_INET6, 7, "default via fd6f:36f8:3005::1 dev bds")
+		})
+	}
 }

libnetwork/controller.go

@@ -155,10 +155,11 @@ func New(cfgOptions ...config.Option) (*Controller, error) {
 	// generate many unnecessary warnings
 	c.reservePools()
 
-	// Cleanup resources
-	if err := c.sandboxCleanup(c.cfg.ActiveSandboxes); err != nil {
+	if err := c.sandboxRestore(c.cfg.ActiveSandboxes); err != nil {
 		log.G(context.TODO()).WithError(err).Error("error during sandbox cleanup")
 	}
+
+	// Cleanup resources
 	if err := c.cleanupLocalEndpoints(); err != nil {
 		log.G(context.TODO()).WithError(err).Warnf("error during endpoint cleanup")
 	}

libnetwork/osl/namespace_linux.go

@@ -398,8 +398,8 @@ func (n *Namespace) Destroy() error {
 	return nil
 }
 
-// Restore restores the network namespace.
-func (n *Namespace) Restore(interfaces map[Iface][]IfaceOption, routes []*types.StaticRoute, gw net.IP, gw6 net.IP) error {
+// RestoreInterfaces restores the network namespace's interfaces.
+func (n *Namespace) RestoreInterfaces(interfaces map[Iface][]IfaceOption) error {
 	// restore interfaces
 	for iface, opts := range interfaces {
 		i, err := newInterface(n, iface.SrcName, iface.DstPrefix, opts...)
@@ -422,18 +422,31 @@ func (n *Namespace) Restore(interfaces map[Iface][]IfaceOption, routes []*types.
 					break
 				}
 				// find the interface name by ip
+				findIfname := func(needle *net.IPNet, haystack []netlink.Addr) (string, bool) {
+					for _, addr := range haystack {
+						if addr.IPNet.String() == needle.String() {
+							return ifaceName, true
+						}
+					}
+					return "", false
+				}
 				if i.address != nil {
 					addresses, err := n.nlHandle.AddrList(link, netlink.FAMILY_V4)
 					if err != nil {
 						return err
 					}
-					for _, addr := range addresses {
-						if addr.IPNet.String() == i.address.String() {
-							i.dstName = ifaceName
-							break
-						}
+					if name, found := findIfname(i.address, addresses); found {
+						i.dstName = name
+						break
 					}
-					if i.dstName == ifaceName {
+				}
+				if i.addressIPv6 != nil {
+					addresses, err := n.nlHandle.AddrList(link, netlink.FAMILY_V6)
+					if err != nil {
+						return err
+					}
+					if name, found := findIfname(i.address, addresses); found {
+						i.dstName = name
 						break
 					}
 				}
@@ -459,18 +472,34 @@ func (n *Namespace) Restore(interfaces map[Iface][]IfaceOption, routes []*types.
 			n.mu.Unlock()
 		}
 	}
+	return nil
+}
 
-	// restore routes and gateways
+func (n *Namespace) RestoreRoutes(routes []*types.StaticRoute) {
 	n.mu.Lock()
+	defer n.mu.Unlock()
 	n.staticRoutes = append(n.staticRoutes, routes...)
-	if len(gw) > 0 {
-		n.gw = gw
+}
+
+func (n *Namespace) RestoreGateway(ipv4 bool, gw net.IP, srcName string) {
+	n.mu.Lock()
+	defer n.mu.Unlock()
+
+	if gw == nil {
+		// There's no gateway address, so the default route is bound to the interface.
+		if ipv4 {
+			n.defRoute4SrcName = srcName
+		} else {
+			n.defRoute6SrcName = srcName
+		}
+		return
 	}
-	if len(gw6) > 0 {
-		n.gwv6 = gw6
+
+	if ipv4 {
+		n.gw = gw
+	} else {
+		n.gwv6 = gw
 	}
-	n.mu.Unlock()
-	return nil
 }
 
 // IPv6LoEnabled returns true if the loopback interface had an IPv6 address when

libnetwork/sandbox_linux.go

@@ -283,15 +283,18 @@ func (sb *Sandbox) restoreOslSandbox() error {
 		}
 	}
 
-	gwep4, gwep6 := sb.getGatewayEndpoint()
-	if gwep4 != nil {
-		if err := sb.osSbox.Restore(interfaces, routes, gwep4.joinInfo.gw, gwep4.joinInfo.gw6); err != nil {
-			return err
-		}
+	if err := sb.osSbox.RestoreInterfaces(interfaces); err != nil {
+		return err
 	}
-	if gwep6 != nil {
-		if err := sb.osSbox.Restore(interfaces, routes, gwep6.joinInfo.gw, gwep6.joinInfo.gw6); err != nil {
-			return err
+	if len(routes) > 0 {
+		sb.osSbox.RestoreRoutes(routes)
+	}
+	if gwEp4, gwEp6 := sb.getGatewayEndpoint(); gwEp4 != nil || gwEp6 != nil {
+		if gwEp4 != nil {
+			sb.osSbox.RestoreGateway(true, gwEp4.joinInfo.gw, gwEp4.iface.srcName)
+		}
+		if gwEp6 != nil {
+			sb.osSbox.RestoreGateway(false, gwEp6.joinInfo.gw6, gwEp6.iface.srcName)
 		}
 	}
 

libnetwork/sandbox_store.go

@@ -167,7 +167,8 @@ func (sb *Sandbox) storeDelete() error {
 	})
 }
 
-func (c *Controller) sandboxCleanup(activeSandboxes map[string]interface{}) error {
+// sandboxRestore restores Sandbox objects from the store, deleting them if they're not active.
+func (c *Controller) sandboxRestore(activeSandboxes map[string]interface{}) error {
 	sandboxStates, err := c.store.List(&sbState{c: c})
 	if err != nil {
 		if err == datastore.ErrKeyNotFound {
@@ -183,6 +184,7 @@ func (c *Controller) sandboxCleanup(activeSandboxes map[string]interface{}) erro
 			id:                 sbs.ID,
 			controller:         sbs.c,
 			containerID:        sbs.Cid,
+			epPriority:         sbs.EpPriority,
 			extDNS:             sbs.ExtDNS,
 			endpoints:          []*Endpoint{},
 			populatedEndpoints: map[string]struct{}{},
@@ -255,6 +257,10 @@ func (c *Controller) sandboxCleanup(activeSandboxes map[string]interface{}) erro
 			continue
 		}
 
+		for _, ep := range sb.endpoints {
+			sb.populatedEndpoints[ep.id] = struct{}{}
+		}
+
 		// reconstruct osl sandbox field
 		if !sb.config.useDefaultSandBox {
 			if err := sb.restoreOslSandbox(); err != nil {