From a0bbf0c3aa067499d36a7b9c222039b16d040f90 Mon Sep 17 00:00:00 2001 From: Daman Arora Date: Mon, 25 Mar 2024 02:04:38 +0530 Subject: [PATCH] allow primary nic for secondary bridge If primary nic is configured with secondary bridge antrea-agent will move primary nic configuration(IPs and Routes) to the secondary bridge, and will revert the change on shutdown. Signed-off-by: Daman Arora --- cmd/antrea-agent/agent.go | 1 + pkg/agent/agent_linux.go | 176 +++--------------------- pkg/agent/agent_linux_test.go | 9 -- pkg/agent/secondarynetwork/init.go | 68 ++++++--- pkg/agent/secondarynetwork/init_test.go | 77 +++++++---- pkg/agent/util/net.go | 2 +- pkg/agent/util/net_linux.go | 155 ++++++++++++++++++++- pkg/agent/util/net_windows.go | 7 + 8 files changed, 277 insertions(+), 218 deletions(-) diff --git a/cmd/antrea-agent/agent.go b/cmd/antrea-agent/agent.go index 7c663f57e89..35fb7e32909 100644 --- a/cmd/antrea-agent/agent.go +++ b/cmd/antrea-agent/agent.go @@ -728,6 +728,7 @@ func run(o *Options) error { } if features.DefaultFeatureGate.Enabled(features.SecondaryNetwork) { + defer secondarynetwork.RestoreHostInterfaceConfiguration(&o.config.SecondaryNetwork) if err := secondarynetwork.Initialize( o.config.ClientConnection, o.config.KubeAPIServerOverride, k8sClient, localPodInformer.Get(), nodeConfig.Name, diff --git a/pkg/agent/agent_linux.go b/pkg/agent/agent_linux.go index bc0b4fbeca9..79c3578926a 100644 --- a/pkg/agent/agent_linux.go +++ b/pkg/agent/agent_linux.go @@ -23,7 +23,6 @@ import ( "net" "time" - "github.com/vishvananda/netlink" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/klog/v2" @@ -38,9 +37,6 @@ var ( // getInterfaceByName is meant to be overridden for testing. getInterfaceByName = net.InterfaceByName - // getAllIPNetsByName is meant to be overridden for testing. - getAllIPNetsByName = util.GetAllIPNetsByName - // setInterfaceARPAnnounce is meant to be overridden for testing. setInterfaceARPAnnounce = util.EnsureARPAnnounceOnInterface ) @@ -70,22 +66,10 @@ func (i *Initializer) prepareOVSBridgeForK8sNode() error { uplinkNetConfig := i.nodeConfig.UplinkNetConfig uplinkNetConfig.Name = adapter.Name uplinkNetConfig.MAC = adapter.HardwareAddr - uplinkIPs, err := getAllIPNetsByName(adapter.Name) - if err != nil { - return fmt.Errorf("failed to get uplink IPs: %w", err) - } - uplinkNetConfig.IPs = uplinkIPs - uplinkNetConfig.Index = adapter.Index // Gateway and DNSServers are not configured at adapter in Linux // Limitation: dynamic DNS servers will be lost after DHCP lease expired uplinkNetConfig.Gateway = "" uplinkNetConfig.DNSServers = "" - // Save routes which are configured on the uplink interface. - // The routes on the host will be lost when moving the network configuration of the uplink interface - // to the OVS bridge local interface. The saved routes will be restored on host after that. - if err = i.saveHostRoutes(); err != nil { - return err - } // Set datapathID of OVS bridge. // If no datapathID configured explicitly, the reconfiguration operation will change OVS bridge datapathID @@ -140,56 +124,6 @@ func getTransportIPNetDeviceByName(ifaceName string, ovsBridgeName string) (*net return util.GetIPNetDeviceByName(ifaceName) } -// saveHostRoutes saves the routes which were configured on the uplink interface -// before the interface is configured as the OVS brdige uplink. These routes -// will be moved to the bridge interface together with the interface IP -// configuration. -func (i *Initializer) saveHostRoutes() error { - routes, err := netlink.RouteList(nil, netlink.FAMILY_V4) - if err != nil { - return err - } - for _, route := range routes { - if route.LinkIndex != i.nodeConfig.UplinkNetConfig.Index { - klog.V(2).Infof("Skipped host route not on uplink: %+v", route) - continue - } - // Skip IPv6 routes until we support IPv6 stack. - // TODO(gran): support IPv6 - if route.Gw.To4() == nil { - klog.V(2).Infof("Skipped IPv6 host route: %+v", route) - continue - } - klog.Infof("Got host route=%+v", route) - i.nodeConfig.UplinkNetConfig.Routes = append(i.nodeConfig.UplinkNetConfig.Routes, route) - } - return nil -} - -// restoreHostRoutes restores the host routes which are lost when moving the IP -// configuration of uplink interface to the OVS bridge interface during -// the Antrea bridge initialization stage. -// The backup routes are restored after the IP configuration changes. -func (i *Initializer) restoreHostRoutes() error { - return i.restoreHostRoutesToInterface(i.nodeConfig.UplinkNetConfig.Name) -} - -func (i *Initializer) restoreHostRoutesToInterface(ifaceName string) error { - iface, err := net.InterfaceByName(ifaceName) - if err != nil { - return nil - } - for _, routeInterface := range i.nodeConfig.UplinkNetConfig.Routes { - route := routeInterface.(netlink.Route) - newRoute := route - newRoute.LinkIndex = iface.Index - if err := netlink.RouteReplace(&newRoute); err != nil { - return err - } - } - return nil -} - func (i *Initializer) ConnectUplinkToOVSBridge() error { // Return immediately on Linux if connectUplinkToBridge is false. if !i.connectUplinkToBridge { @@ -198,18 +132,21 @@ func (i *Initializer) ConnectUplinkToOVSBridge() error { klog.InfoS("Bridging uplink to OVS bridge") var err error uplinkNetConfig := i.nodeConfig.UplinkNetConfig - uplinkName := uplinkNetConfig.Name - bridgedUplinkName := util.GenerateUplinkInterfaceName(uplinkNetConfig.Name) - uplinkIPs := uplinkNetConfig.IPs - // If the uplink port already exists, just return. - if uplinkOFPort, err := i.ovsBridgeClient.GetOFPort(bridgedUplinkName, false); err == nil { - klog.InfoS("Uplink already exists, skip the configuration", "uplink", bridgedUplinkName, "port", uplinkOFPort) - return nil + externalIDs := map[string]interface{}{ + interfacestore.AntreaInterfaceTypeKey: interfacestore.AntreaHost, } - - if err := util.RenameInterface(uplinkName, bridgedUplinkName); err != nil { - return fmt.Errorf("failed to change uplink interface name: err=%w", err) + bridgedUplinkName, exists, err := util.PrepareHostInterfaceConnection( + i.ovsBridgeClient, + uplinkNetConfig.Name, + int32(i.nodeConfig.HostInterfaceOFPort), + externalIDs, + ) + if err != nil { + return err + } + if exists { + return nil } // Create uplink port. @@ -226,64 +163,6 @@ func (i *Initializer) ConnectUplinkToOVSBridge() error { klog.InfoS("Allocated OpenFlow port for uplink interface", "port", bridgedUplinkName, "ofPort", uplinkOFPort) uplinkInterface.OVSPortConfig = &interfacestore.OVSPortConfig{uplinkPortUUID, uplinkOFPort} //nolint: govet i.ifaceStore.AddInterface(uplinkInterface) - - // Create local port. - externalIDs := map[string]interface{}{ - interfacestore.AntreaInterfaceTypeKey: interfacestore.AntreaHost, - } - if _, err = i.ovsBridgeClient.CreateInternalPort(uplinkName, int32(i.nodeConfig.HostInterfaceOFPort), uplinkNetConfig.MAC.String(), externalIDs); err != nil { - return fmt.Errorf("cannot create host interface port %s: err=%w", uplinkName, err) - } - - // Move network configuration of uplink interface to OVS bridge local interface. - // The net configuration of uplink will be restored by RestoreOVSBridge when shutting down. - wait.PollUntilContextTimeout(context.TODO(), 100*time.Millisecond, 10000*time.Millisecond, true, - func(ctx context.Context) (bool, error) { - // Wait a few seconds for OVS bridge local port. - link, err := netlink.LinkByName(uplinkName) - if err != nil { - klog.V(4).InfoS("OVS bridge local port is not ready", "port", uplinkName, "err", err) - return false, nil - } - klog.InfoS("OVS bridge local port is ready", "type", link.Type(), "attrs", link.Attrs()) - return true, nil - }) - localLink, err := netlink.LinkByName(uplinkName) - if err != nil { - return err - } - if _, _, err = util.SetLinkUp(uplinkName); err != nil { - return err - } - - // Check if uplink is configured with an IPv6 address: if it is, we need to ensure that IPv6 - // is enabled on the OVS internal port as we need to move all IP addresses over. - uplinkHasIPv6Address := false - for _, ip := range uplinkIPs { - if ip.IP.To4() == nil { - uplinkHasIPv6Address = true - break - } - } - if uplinkHasIPv6Address { - klog.InfoS("Uplink has IPv6 address, ensuring that IPv6 is enabled on bridge local port", "port", uplinkName) - if err := util.EnsureIPv6EnabledOnInterface(uplinkName); err != nil { - klog.ErrorS(err, "Failed to ensure that IPv6 is enabled on bridge local port, moving uplink IPs to bridge is likely to fail", "port", uplinkName) - } - } - - if err = util.ConfigureLinkAddresses(localLink.Attrs().Index, uplinkIPs); err != nil { - return err - } - if err = util.ConfigureLinkAddresses(uplinkNetConfig.Index, nil); err != nil { - return err - } - // Restore the host routes which are lost when moving the network configuration of the - // uplink interface to OVS bridge interface. - if err = i.restoreHostRoutes(); err != nil { - return err - } - return nil } @@ -294,34 +173,11 @@ func (i *Initializer) RestoreOVSBridge() { return } klog.InfoS("Restoring bridge config to uplink...") - uplinkNetConfig := i.nodeConfig.UplinkNetConfig - uplinkName := "" - bridgedUplinkName := "" - if uplinkNetConfig != nil { - uplinkName = uplinkNetConfig.Name - bridgedUplinkName = util.GenerateUplinkInterfaceName(uplinkName) - } - brName := i.ovsBridge - if uplinkName != "" { - uplinkIPs := uplinkNetConfig.IPs - if err := util.DeleteOVSPort(brName, uplinkName); err != nil { - klog.ErrorS(err, "Delete OVS port failed", "port", uplinkName) - } - if err := util.DeleteOVSPort(brName, bridgedUplinkName); err != nil { - klog.ErrorS(err, "Delete OVS port failed", "port", bridgedUplinkName) - } - if err := util.RenameInterface(bridgedUplinkName, uplinkName); err != nil { - klog.ErrorS(err, "Restore uplink name failed", "uplink", bridgedUplinkName) - } - if err := util.ConfigureLinkAddresses(uplinkNetConfig.Index, uplinkIPs); err != nil { - klog.ErrorS(err, "Configure IP to uplink failed", "uplink", uplinkName) - } - if err := i.restoreHostRoutesToInterface(uplinkName); err != nil { - klog.ErrorS(err, "Configure route to uplink interface failed", "uplink", uplinkName) - } + if i.nodeConfig.UplinkNetConfig.Name != "" { + util.RestoreHostInterfaceConfiguration(i.ovsBridge, i.nodeConfig.UplinkNetConfig.Name) + klog.InfoS("Finished restoring bridge config to uplink...") } - klog.InfoS("Finished to restore bridge config to uplink...") } func (i *Initializer) setInterfaceMTU(iface string, mtu int) error { diff --git a/pkg/agent/agent_linux_test.go b/pkg/agent/agent_linux_test.go index b814009f2ef..ac0bb63add9 100644 --- a/pkg/agent/agent_linux_test.go +++ b/pkg/agent/agent_linux_test.go @@ -41,14 +41,6 @@ func mockGetInterfaceByName(t *testing.T, ipDevice *net.Interface) { t.Cleanup(func() { getInterfaceByName = prevGetInterfaceByName }) } -func mockGetAllIPNetsByName(t *testing.T, ips []*net.IPNet) { - prevGetAllIPNetsByName := getAllIPNetsByName - getAllIPNetsByName = func(name string) ([]*net.IPNet, error) { - return ips, nil - } - t.Cleanup(func() { getAllIPNetsByName = prevGetAllIPNetsByName }) -} - func TestPrepareOVSBridgeForK8sNode(t *testing.T) { macAddr, _ := net.ParseMAC("00:00:5e:00:53:01") _, nodeIPNet, _ := net.ParseCIDR("192.168.10.10/24") @@ -119,7 +111,6 @@ func TestPrepareOVSBridgeForK8sNode(t *testing.T) { initializer.nodeConfig = nodeConfig mockGetIPNetDeviceFromIP(t, nodeIPNet, ipDevice) mockGetInterfaceByName(t, ipDevice) - mockGetAllIPNetsByName(t, []*net.IPNet{nodeIPNet}) if tt.expectedCalls != nil { tt.expectedCalls(mockOVSBridgeClient) } diff --git a/pkg/agent/secondarynetwork/init.go b/pkg/agent/secondarynetwork/init.go index 7333b6e5878..589ce60004f 100644 --- a/pkg/agent/secondarynetwork/init.go +++ b/pkg/agent/secondarynetwork/init.go @@ -27,6 +27,7 @@ import ( "antrea.io/antrea/pkg/agent/interfacestore" "antrea.io/antrea/pkg/agent/secondarynetwork/podwatch" + "antrea.io/antrea/pkg/agent/util" agentconfig "antrea.io/antrea/pkg/config/agent" "antrea.io/antrea/pkg/ovs/ovsconfig" "antrea.io/antrea/pkg/util/channel" @@ -48,13 +49,37 @@ func Initialize( nodeName string, podUpdateSubscriber channel.Subscriber, stopCh <-chan struct{}, - config *agentconfig.SecondaryNetworkConfig, ovsdb *ovsdb.OVSDB) error { + secNetConfig *agentconfig.SecondaryNetworkConfig, ovsdb *ovsdb.OVSDB) error { - ovsBridgeClient, err := createOVSBridge(config.OVSBridges, ovsdb) + ovsBridgeClient, err := createOVSBridge(secNetConfig.OVSBridges, ovsdb) if err != nil { return err } + // We only support moving and restoring of interface configuration to OVS Bridge for the single physical interface case. + if len(secNetConfig.OVSBridges) != 0 { + phyInterfaces := make([]string, len(secNetConfig.OVSBridges[0].PhysicalInterfaces)) + copy(phyInterfaces, secNetConfig.OVSBridges[0].PhysicalInterfaces) + if len(phyInterfaces) == 1 { + + bridgedName, _, err := util.PrepareHostInterfaceConnection( + ovsBridgeClient, + phyInterfaces[0], + 0, + map[string]interface{}{ + interfacestore.AntreaInterfaceTypeKey: interfacestore.AntreaHost, + }, + ) + if err != nil { + return err + } + phyInterfaces[0] = bridgedName + } + if err = connectPhyInterfacesToOVSBridge(ovsBridgeClient, phyInterfaces); err != nil { + return err + } + } + // Create the NetworkAttachmentDefinition client, which handles access to secondary network object // definition from the API Server. netAttachDefClient, err := createNetworkAttachDefClient(clientConnectionConfig, kubeAPIServerOverride) @@ -74,38 +99,49 @@ func Initialize( return nil } -// TODO: check and update bridge configuration. +// RestoreHostInterfaceConfiguration restores interface configuration from secondary-bridge back to host-interface. +func RestoreHostInterfaceConfiguration(secNetConfig *agentconfig.SecondaryNetworkConfig) { + if len(secNetConfig.OVSBridges[0].PhysicalInterfaces) == 1 { + util.RestoreHostInterfaceConfiguration(secNetConfig.OVSBridges[0].BridgeName, secNetConfig.OVSBridges[0].PhysicalInterfaces[0]) + } +} + func createOVSBridge(bridges []agentconfig.OVSBridgeConfig, ovsdb *ovsdb.OVSDB) (ovsconfig.OVSBridgeClient, error) { if len(bridges) == 0 { return nil, nil } // Only one OVS bridge is supported. bridgeConfig := bridges[0] - - for _, phyInterface := range bridgeConfig.PhysicalInterfaces { - if _, err := interfaceByNameFn(phyInterface); err != nil { - return nil, fmt.Errorf("failed to get interface %s: %v", phyInterface, err) - } - } - ovsBridgeClient := newOVSBridgeFn(bridgeConfig.BridgeName, ovsconfig.OVSDatapathSystem, ovsdb) if err := ovsBridgeClient.Create(); err != nil { return nil, fmt.Errorf("failed to create OVS bridge %s: %v", bridgeConfig.BridgeName, err) } klog.InfoS("OVS bridge created", "bridge", bridgeConfig.BridgeName) + return ovsBridgeClient, nil +} + +func connectPhyInterfacesToOVSBridge(ovsBridgeClient ovsconfig.OVSBridgeClient, phyInterfaces []string) error { + for _, phyInterface := range phyInterfaces { + if _, err := interfaceByNameFn(phyInterface); err != nil { + return fmt.Errorf("failed to get interface %s: %v", phyInterface, err) + } + } - for i, phyInterface := range bridgeConfig.PhysicalInterfaces { + externalIDs := map[string]interface{}{ + interfacestore.AntreaInterfaceTypeKey: interfacestore.AntreaUplink, + } + for i, phyInterface := range phyInterfaces { if _, err := ovsBridgeClient.GetOFPort(phyInterface, false); err == nil { - klog.V(2).InfoS("Physical interface already connected to OVS bridge, skip the configuration", "device", phyInterface, "bridge", bridgeConfig.BridgeName) + klog.V(2).InfoS("Physical interface already connected to secondary OVS bridge, skip the configuration", "device", phyInterface) continue } - if _, err := ovsBridgeClient.CreateUplinkPort(phyInterface, int32(i), map[string]interface{}{interfacestore.AntreaInterfaceTypeKey: interfacestore.AntreaUplink}); err != nil { - return nil, fmt.Errorf("failed to create OVS uplink port %s: %v", phyInterface, err) + if _, err := ovsBridgeClient.CreateUplinkPort(phyInterface, int32(i), externalIDs); err != nil { + return fmt.Errorf("failed to create OVS uplink port %s: %v", phyInterface, err) } - klog.InfoS("Physical interface added to OVS bridge", "device", phyInterface, "bridge", bridgeConfig.BridgeName) + klog.InfoS("Physical interface added to secondary OVS bridge", "device", phyInterface) } - return ovsBridgeClient, nil + return nil } // CreateNetworkAttachDefClient creates net-attach-def client handle from the given config. diff --git a/pkg/agent/secondarynetwork/init_test.go b/pkg/agent/secondarynetwork/init_test.go index b9be6fe0fd2..49de49a7afd 100644 --- a/pkg/agent/secondarynetwork/init_test.go +++ b/pkg/agent/secondarynetwork/init_test.go @@ -33,11 +33,10 @@ const nonExistingInterface = "non-existing" func TestCreateOVSBridge(t *testing.T) { tests := []struct { - name string - ovsBridges []string - physicalInterfaces []string - expectedErr string - expectedCalls func(m *ovsconfigtest.MockOVSBridgeClient) + name string + ovsBridges []string + expectedErr string + expectedCalls func(m *ovsconfigtest.MockOVSBridgeClient) }{ { name: "no bridge", @@ -64,22 +63,56 @@ func TestCreateOVSBridge(t *testing.T) { m.EXPECT().Create().Return(ovsconfig.InvalidArgumentsError("create error")) }, }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + var bridges []agentconfig.OVSBridgeConfig + for _, brName := range tc.ovsBridges { + br := agentconfig.OVSBridgeConfig{BridgeName: brName} + bridges = append(bridges, br) + } + + controller := mock.NewController(t) + mockOVSBridgeClient := ovsconfigtest.NewMockOVSBridgeClient(controller) + + mockNewOVSBridge(t, mockOVSBridgeClient) + if tc.expectedCalls != nil { + tc.expectedCalls(mockOVSBridgeClient) + } + + brClient, err := createOVSBridge(bridges, nil) + if tc.expectedErr != "" { + assert.ErrorContains(t, err, tc.expectedErr) + assert.Nil(t, brClient) + } else { + require.NoError(t, err) + if tc.expectedCalls != nil { + assert.NotNil(t, brClient) + } + } + }) + } +} + +func TestConnectPhyInterfacesToOVSBridge(t *testing.T) { + tests := []struct { + name string + physicalInterfaces []string + expectedErr string + expectedCalls func(m *ovsconfigtest.MockOVSBridgeClient) + }{ { name: "one interface", - ovsBridges: []string{"br1"}, - physicalInterfaces: []string{"eth1"}, + physicalInterfaces: []string{"eth0~"}, expectedCalls: func(m *ovsconfigtest.MockOVSBridgeClient) { - m.EXPECT().Create().Return(nil) - m.EXPECT().GetOFPort("eth1", false).Return(int32(0), ovsconfig.InvalidArgumentsError("port not found")) - m.EXPECT().CreateUplinkPort("eth1", int32(0), map[string]interface{}{"antrea-type": "uplink"}).Return("", nil) + m.EXPECT().GetOFPort("eth0~", false).Return(int32(0), ovsconfig.InvalidArgumentsError("port not found")) + m.EXPECT().CreateUplinkPort("eth0~", int32(0), map[string]interface{}{"antrea-type": "uplink"}).Return("", nil) }, }, { name: "two interfaces", - ovsBridges: []string{"br1", "br2"}, physicalInterfaces: []string{"eth1", "eth2"}, expectedCalls: func(m *ovsconfigtest.MockOVSBridgeClient) { - m.EXPECT().Create().Return(nil) m.EXPECT().GetOFPort("eth1", false).Return(int32(0), ovsconfig.InvalidArgumentsError("port not found")) m.EXPECT().CreateUplinkPort("eth1", int32(0), map[string]interface{}{"antrea-type": "uplink"}).Return("", nil) m.EXPECT().GetOFPort("eth2", false).Return(int32(1), ovsconfig.InvalidArgumentsError("port not found")) @@ -88,26 +121,21 @@ func TestCreateOVSBridge(t *testing.T) { }, { name: "interface already attached", - ovsBridges: []string{"br1"}, physicalInterfaces: []string{"eth1"}, expectedCalls: func(m *ovsconfigtest.MockOVSBridgeClient) { - m.EXPECT().Create().Return(nil) m.EXPECT().GetOFPort("eth1", false).Return(int32(0), nil) }, }, { name: "non-existing interface", - ovsBridges: []string{"br1"}, physicalInterfaces: []string{nonExistingInterface, "eth2"}, expectedErr: "failed to get interface", }, { name: "create port error", - ovsBridges: []string{"br1"}, physicalInterfaces: []string{"eth1"}, expectedErr: "create error", expectedCalls: func(m *ovsconfigtest.MockOVSBridgeClient) { - m.EXPECT().Create().Return(nil) m.EXPECT().GetOFPort("eth1", false).Return(int32(0), ovsconfig.InvalidArgumentsError("port not found")) m.EXPECT().CreateUplinkPort("eth1", int32(0), map[string]interface{}{"antrea-type": "uplink"}).Return("", ovsconfig.InvalidArgumentsError("create error")) }, @@ -115,31 +143,20 @@ func TestCreateOVSBridge(t *testing.T) { } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - var bridges []agentconfig.OVSBridgeConfig - for _, brName := range tc.ovsBridges { - br := agentconfig.OVSBridgeConfig{BridgeName: brName} - br.PhysicalInterfaces = tc.physicalInterfaces - bridges = append(bridges, br) - } controller := mock.NewController(t) mockOVSBridgeClient := ovsconfigtest.NewMockOVSBridgeClient(controller) - mockNewOVSBridge(t, mockOVSBridgeClient) mockInterfaceByName(t) if tc.expectedCalls != nil { tc.expectedCalls(mockOVSBridgeClient) } - brClient, err := createOVSBridge(bridges, nil) + err := connectPhyInterfacesToOVSBridge(mockOVSBridgeClient, tc.physicalInterfaces) if tc.expectedErr != "" { assert.ErrorContains(t, err, tc.expectedErr) - assert.Nil(t, brClient) } else { - require.NoError(t, err) - if tc.expectedCalls != nil { - assert.NotNil(t, brClient) - } + assert.NoError(t, err) } }) } diff --git a/pkg/agent/util/net.go b/pkg/agent/util/net.go index c830c0a4aa1..39504461ba0 100644 --- a/pkg/agent/util/net.go +++ b/pkg/agent/util/net.go @@ -441,7 +441,7 @@ func GetIPNetsByLink(link *net.Interface) ([]*net.IPNet, error) { } var addrs []*net.IPNet for _, a := range addrList { - if ipNet, ok := a.(*net.IPNet); ok { + if ipNet, ok := a.(*net.IPNet); ok && !ipNet.IP.IsLinkLocalUnicast() { addrs = append(addrs, ipNet) } } diff --git a/pkg/agent/util/net_linux.go b/pkg/agent/util/net_linux.go index 301691fb717..3c170a85eab 100644 --- a/pkg/agent/util/net_linux.go +++ b/pkg/agent/util/net_linux.go @@ -35,6 +35,7 @@ import ( utilnetlink "antrea.io/antrea/pkg/agent/util/netlink" "antrea.io/antrea/pkg/agent/util/sysctl" + "antrea.io/antrea/pkg/ovs/ovsconfig" ) var ( @@ -240,8 +241,9 @@ func SetAdapterMACAddress(adapterName string, macConfig *net.HardwareAddr) error return netlinkUtil.LinkSetHardwareAddr(link, *macConfig) } -// DeleteOVSPort deletes specific OVS port. This function calls ovs-vsctl command to bypass OVS bridge client to work when agent exiting. -func DeleteOVSPort(brName, portName string) error { +// deleteOVSPort deletes specific OVS port. This function calls ovs-vsctl command to bypass +// OVS bridge client to work when agent exiting. +func deleteOVSPort(brName, portName string) error { cmd := exec.Command("ovs-vsctl", "--if-exists", "del-port", brName, portName) return cmd.Run() } @@ -376,3 +378,152 @@ func renameHostInterface(oriName string, newName string) error { } return nil } + +func interfaceExists(name string) (bool, error) { + intfs, err := netInterfaces() + if err != nil { + return false, err + } + for _, intf := range intfs { + if intf.Name == name { + return true, nil + } + } + return false, nil +} + +// PrepareHostInterfaceConnection prepares host interface connection to the OVS bridge client by: +// 1. Renaming the host interface (a bridged suffix will be added to it). +// 2. Creating an internal port (original name of the host interface will be used here). +// 3. Moving IPs of host interface to this new link/internal-port. +// 4. Moving routes of host interface to the new link/internal-port. +// and returns the bridged name, true if it already exists, and error. +func PrepareHostInterfaceConnection( + bridge ovsconfig.OVSBridgeClient, + ifaceName string, + ifaceOFPort int32, + externalIDs map[string]interface{}, +) (string, bool, error) { + bridgedName := GenerateUplinkInterfaceName(ifaceName) + // If the port already exists, just return. + if ofPort, err := bridge.GetOFPort(bridgedName, false); err == nil { + klog.InfoS("Port already exists, skip the configuration", "port", bridgedName, "ofPort", ofPort) + return "", true, nil + } + + iface, ifaceIPs, ifaceRoutes, err := GetInterfaceConfig(ifaceName) + if err != nil { + return "", false, nil + } + + if err = RenameInterface(ifaceName, bridgedName); err != nil { + return "", false, err + } + if _, err = bridge.CreateInternalPort(ifaceName, ifaceOFPort, iface.HardwareAddr.String(), externalIDs); err != nil { + return "", false, fmt.Errorf("failed to create internal port: %v", err) + } + + // Wait a few seconds for OVS bridge local port. + if err = wait.PollUntilContextTimeout(context.TODO(), 100*time.Millisecond, 10*time.Second, true, func(ctx context.Context) (bool, error) { + link, err := netlink.LinkByName(ifaceName) + if err != nil { + klog.V(4).InfoS("OVS bridge local port is not ready", "port", ifaceName, "err", err) + return false, nil + } + klog.InfoS("OVS bridge local port is ready", "type", link.Type(), "attrs", link.Attrs()) + return true, nil + }); err != nil { + return "", false, fmt.Errorf("failed waiting for internal port to show up: %v", err) + } + + localLink, err := netlink.LinkByName(ifaceName) + if err != nil { + return "", false, err + } + if _, _, err = SetLinkUp(ifaceName); err != nil { + return "", false, fmt.Errorf("failed to set link up: %v", err) + } + + // Check if interface is configured with an IPv6 address: if it is, we need to ensure that IPv6 + // is enabled on the OVS internal port as we need to move all IP addresses over. + for _, ip := range ifaceIPs { + if ip.IP.To4() == nil { + klog.InfoS("Interface has IPv6 address, ensuring that IPv6 is enabled on bridge local port", "port", ifaceName) + if err := EnsureIPv6EnabledOnInterface(ifaceName); err != nil { + klog.ErrorS(err, "Failed to ensure that IPv6 is enabled on bridge local port, moving uplink IPs to bridge is likely to fail", "port", ifaceName) + } + break + } + } + + if err = ConfigureLinkAddresses(localLink.Attrs().Index, ifaceIPs); err != nil { + return "", false, err + } + if err = ConfigureLinkAddresses(iface.Index, nil); err != nil { + return "", false, err + } + // Restore the host routes which are lost when moving the network configuration of the + // host interface to OVS bridge interface. + if err = ConfigureLinkRoutes(localLink, ifaceRoutes); err != nil { + return "", false, err + } + return bridgedName, false, nil +} + +// RestoreHostInterfaceConfiguration restore the configuration from bridge back to host interface, reverting the +// actions taken in PrepareHostInterfaceConnection. +func RestoreHostInterfaceConfiguration(brName string, interfaceName string) { + klog.V(4).InfoS("Restoring bridge config to host interface") + bridgedName := GenerateUplinkInterfaceName(interfaceName) + // restore if interface eth0~ exists + if exists, err := interfaceExists(bridgedName); err != nil { + klog.ErrorS(err, "Failed to check if interface exists", "interface", bridgedName) + return + } else if exists { + // get interface config + var interfaceIPs []*net.IPNet + var interfaceRoutes []interface{} + if exists, err = interfaceExists(interfaceName); err != nil { + klog.ErrorS(err, "Failed to check if interface exists", "interface", interfaceName) + } else if exists { + _, interfaceIPs, interfaceRoutes, err = GetInterfaceConfig(interfaceName) + if err != nil { + klog.ErrorS(err, "Failed to get interface config", "interface", interfaceName) + } + + // delete internal port (eth0) + if err = deleteOVSPort(brName, interfaceName); err != nil { + klog.ErrorS(err, "Delete OVS port failed", "port", bridgedName) + } + } + // remove host interface (eth0~) from bridge + if err = deleteOVSPort(brName, bridgedName); err != nil { + klog.ErrorS(err, "Delete OVS port failed", "port", bridgedName) + return + } + + // rename host interface(eth0~ -> eth0) + if err = RenameInterface(bridgedName, interfaceName); err != nil { + klog.ErrorS(err, "Restore host interface name failed", "from", bridgedName, "to", interfaceName) + } else { + var link netlink.Link + if link, err = netlink.LinkByName(interfaceName); err != nil { + klog.ErrorS(err, "Failed to get link", "interface", interfaceName) + } else { + if len(interfaceIPs) > 0 { + // restore IPs to eth0 + if err = ConfigureLinkAddresses(link.Attrs().Index, interfaceIPs); err != nil { + klog.ErrorS(err, "Restore IPs to host interface failed", "interface", interfaceName) + } + } + if len(interfaceRoutes) > 0 { + // restore routes to eth0 + if err = ConfigureLinkRoutes(link, interfaceRoutes); err != nil { + klog.ErrorS(err, "Restore routes to host interface failed", "interface", interfaceName) + } + } + } + } + klog.V(4).InfoS("Finished restoring bridge config to host interface", "interface", interfaceName, "bridge", brName) + } +} diff --git a/pkg/agent/util/net_windows.go b/pkg/agent/util/net_windows.go index dc57c0a2dbf..ae73dc89962 100644 --- a/pkg/agent/util/net_windows.go +++ b/pkg/agent/util/net_windows.go @@ -43,6 +43,7 @@ import ( ps "antrea.io/antrea/pkg/agent/util/powershell" antreasyscall "antrea.io/antrea/pkg/agent/util/syscall" binding "antrea.io/antrea/pkg/ovs/openflow" + "antrea.io/antrea/pkg/ovs/ovsconfig" iputil "antrea.io/antrea/pkg/util/ip" ) @@ -1325,3 +1326,9 @@ func adapterAddresses() ([]*windows.IpAdapterAddresses, error) { } return aas, nil } + +func PrepareHostInterfaceConnection(_ ovsconfig.OVSBridgeClient, ifaceName string, _ int32, _ map[string]interface{}) (string, bool, error) { + return ifaceName, false, nil +} + +func RestoreHostInterfaceConfiguration(_ string, _ string) {}