From c2c3d32f97235e3203ac94495957cd2fa57d6298 Mon Sep 17 00:00:00 2001 From: Jiajing Hu Date: Sun, 21 Jan 2024 18:02:47 +0800 Subject: [PATCH] Bugfix: Fix incorrect Pod MTU when WireGuard enabled The default tunnel type is GENEVE, when the WireGuard encryption is enabled, the Pod MTU calculator still deducts the GENEVE overhead directly, which causes a MTU error. This patch will calculate all MTU deduction in function CalculateMTUDeduction, including WireGuard and Multicluster. Signed-off-by: Jiajing Hu --- cmd/antrea-agent/agent.go | 2 +- pkg/agent/agent.go | 4 - pkg/agent/config/node_config.go | 29 ++- pkg/agent/config/node_config_test.go | 20 ++ pkg/agent/multicluster/mc_route_controller.go | 2 +- test/e2e/framework.go | 51 +++++ test/e2e/mtu_test.go | 211 ++++++++++++++++++ 7 files changed, 305 insertions(+), 14 deletions(-) create mode 100644 test/e2e/mtu_test.go diff --git a/cmd/antrea-agent/agent.go b/cmd/antrea-agent/agent.go index fba691bdd76..73e37a11cb6 100644 --- a/cmd/antrea-agent/agent.go +++ b/cmd/antrea-agent/agent.go @@ -205,7 +205,7 @@ func run(o *Options) error { IPsecConfig: config.IPsecConfig{ AuthenticationMode: ipsecAuthenticationMode, }, - EnableMulticlusterGW: enableMulticlusterGW, + MulticlusterConfig: o.config.Multicluster, } wireguardConfig := &config.WireGuardConfig{ diff --git a/pkg/agent/agent.go b/pkg/agent/agent.go index 791e18b890f..86b41009869 100644 --- a/pkg/agent/agent.go +++ b/pkg/agent/agent.go @@ -1194,10 +1194,6 @@ func (i *Initializer) getInterfaceMTU(transportInterface *net.Interface) (int, e isIPv6 := i.nodeConfig.NodeIPv6Addr != nil mtu -= i.networkConfig.CalculateMTUDeduction(isIPv6) - - if i.networkConfig.TrafficEncryptionMode == config.TrafficEncryptionModeIPSec { - mtu -= config.IPSecESPOverhead - } return mtu, nil } diff --git a/pkg/agent/config/node_config.go b/pkg/agent/config/node_config.go index ebba7f3da9e..2cca9b1a10e 100644 --- a/pkg/agent/config/node_config.go +++ b/pkg/agent/config/node_config.go @@ -17,7 +17,9 @@ package config import ( "fmt" "net" + "strings" + agentConfig "antrea.io/antrea/pkg/config/agent" "antrea.io/antrea/pkg/ovs/ovsconfig" ) @@ -201,7 +203,7 @@ type NetworkConfig struct { TransportIfaceCIDRs []string IPv4Enabled bool IPv6Enabled bool - // MTUDeduction only counts IPv4 tunnel overhead, no IPsec and WireGuard overhead. + // MTUDeduction is the MTU deduction for encapsulation and encryption in cluster. MTUDeduction int // Set by the defaultMTU config option or auto discovered. // Auto discovery will use MTU value of the Node's transport interface. @@ -209,6 +211,7 @@ type NetworkConfig struct { // encap header. InterfaceMTU int EnableMulticlusterGW bool + MulticlusterConfig agentConfig.MulticlusterConfig } // IsIPv4Enabled returns true if the cluster network supports IPv4. Legal cases are: @@ -265,23 +268,33 @@ func (nc *NetworkConfig) NeedsDirectRoutingToPeer(peerIP net.IP, localIP *net.IP } func (nc *NetworkConfig) CalculateMTUDeduction(isIPv6 bool) int { - var mtuDeduction int + // When WireGuard is enabled, NetworkConfig.TunnelType will be ignored, so we deduct MTU based on WireGuardOverhead. + if nc.TrafficEncryptionMode == TrafficEncryptionModeWireGuard { + nc.MTUDeduction = WireGuardOverhead + return nc.MTUDeduction + } else if nc.TrafficEncryptionMode == TrafficEncryptionModeIPSec { + nc.MTUDeduction = IPSecESPOverhead + } + // When Multi-cluster Gateway is enabled, we need to reduce MTU for potential cross-cluster traffic. if nc.TrafficEncapMode.SupportsEncap() || nc.EnableMulticlusterGW { if nc.TunnelType == ovsconfig.VXLANTunnel { - mtuDeduction = vxlanOverhead + nc.MTUDeduction += vxlanOverhead } else if nc.TunnelType == ovsconfig.GeneveTunnel { - mtuDeduction = geneveOverhead + nc.MTUDeduction += geneveOverhead } else if nc.TunnelType == ovsconfig.GRETunnel { - mtuDeduction = greOverhead + nc.MTUDeduction += greOverhead } } if nc.TrafficEncapMode.SupportsEncap() && isIPv6 { - mtuDeduction += ipv6ExtraOverhead + nc.MTUDeduction += ipv6ExtraOverhead + } + // When multi-cluster WireGuard is enabled, we need to reduce MTU for potential cross-cluster traffic. + if nc.EnableMulticlusterGW && strings.EqualFold(nc.MulticlusterConfig.TrafficEncryptionMode, TrafficEncryptionModeWireGuard.String()) { + nc.MTUDeduction += WireGuardOverhead } - nc.MTUDeduction = mtuDeduction - return mtuDeduction + return nc.MTUDeduction } // ServiceConfig includes K8s Service CIDR and available IP addresses for NodePort. diff --git a/pkg/agent/config/node_config_test.go b/pkg/agent/config/node_config_test.go index 081ce053c7f..662e8ec5de4 100644 --- a/pkg/agent/config/node_config_test.go +++ b/pkg/agent/config/node_config_test.go @@ -20,6 +20,7 @@ import ( "github.com/stretchr/testify/assert" + agentConfig "antrea.io/antrea/pkg/config/agent" "antrea.io/antrea/pkg/ovs/ovsconfig" ) @@ -306,6 +307,25 @@ func TestCalculateMTUDeduction(t *testing.T) { isIPv6: true, expectedMTUDeduction: 70, }, + { + name: "WireGuard enabled", + nc: &NetworkConfig{TrafficEncryptionMode: TrafficEncryptionModeWireGuard}, + expectedMTUDeduction: 80, + }, + { + name: "Multicluster enabled with Geneve encap", + nc: &NetworkConfig{TunnelType: ovsconfig.GeneveTunnel, EnableMulticlusterGW: true}, + expectedMTUDeduction: 50, + }, + { + name: "Geneve encap with Multicluster WireGuard enabled", + nc: &NetworkConfig{ + TunnelType: ovsconfig.GeneveTunnel, + EnableMulticlusterGW: true, + MulticlusterConfig: agentConfig.MulticlusterConfig{TrafficEncryptionMode: "wireGuard"}, + }, + expectedMTUDeduction: 130, + }, } for _, tt := range tests { diff --git a/pkg/agent/multicluster/mc_route_controller.go b/pkg/agent/multicluster/mc_route_controller.go index 568371329d5..e053c698afe 100644 --- a/pkg/agent/multicluster/mc_route_controller.go +++ b/pkg/agent/multicluster/mc_route_controller.go @@ -129,7 +129,7 @@ func NewMCDefaultRouteController( controller.wireGuardConfig = &config.WireGuardConfig{ Port: multiclusterConfig.WireGuard.Port, Name: multiclusterWireGuardInterface, - MTU: controller.nodeConfig.NodeTransportInterfaceMTU - controller.networkConfig.MTUDeduction - config.WireGuardOverhead, + MTU: controller.nodeConfig.NodeTransportInterfaceMTU - controller.networkConfig.MTUDeduction, } } controller.gwInformer.Informer().AddEventHandlerWithResyncPeriod( diff --git a/test/e2e/framework.go b/test/e2e/framework.go index 8eba90cd07f..c862bb26c96 100644 --- a/test/e2e/framework.go +++ b/test/e2e/framework.go @@ -2481,6 +2481,57 @@ func (data *TestData) GetTransportInterface() (string, error) { return "", fmt.Errorf("no interface was assigned with Node IP %s", nodeIP) } +func (data *TestData) GetPodInterfaceName(nodeName string, podName string) (string, error) { + antreaPod, err := data.getAntreaPodOnNode(nodeName) + if err != nil { + return "", fmt.Errorf("failed to get Antrea Pod on Node %s: %v", nodeName, err) + } + cmd := []string{"ip", "-br", "addr", "show"} + stdout, stderr, err := data.RunCommandFromPod(antreaNamespace, antreaPod, agentContainerName, cmd) + if stdout == "" || stderr != "" || err != nil { + return "", fmt.Errorf("failed to show ip address, stdout: %s, stderr: %s, err: %v", stdout, stderr, err) + } + + lines := strings.Split(strings.TrimSpace(stdout), "\n") + for _, line := range lines { + if strings.Contains(line, podName) { + fields := strings.Fields(line) + name, _, _ := strings.Cut(fields[0], "@") + return name, nil + } + } + + return "", fmt.Errorf("no interface was assigned with Pod %s", podName) +} + +func (data *TestData) GetInterfaceMTU(nodeName string, interfaceName string) (int, error) { + antreaPod, err := data.getAntreaPodOnNode(nodeName) + if err != nil { + return 0, fmt.Errorf("failed to get Antrea Pod on Node %s: %v", nodeName, err) + } + cmd := []string{"ip", "link", "show", interfaceName} + stdout, stderr, err := data.RunCommandFromPod(antreaNamespace, antreaPod, agentContainerName, cmd) + if stdout == "" || stderr != "" || err != nil { + return 0, fmt.Errorf("failed to get interface MTU, stdout: %s, stderr: %s, err: %v", stdout, stderr, err) + } + + // Example stdout: + // 22: vxlan-29acf3@if5: mtu 1450 qdisc noqueue master ovs-system state UP mode DEFAULT group default + // link/ether be:41:93:69:87:02 brd ff:ff:ff:ff:ff:ff link-netns cni-320ae61f-5e51-d123-0169-9f1807390500 + fields := strings.Fields(strings.Split(strings.TrimSpace(stdout), "\n")[0]) + // Get the value of "mtu" + for i := 0; i < len(fields)-1; i++ { + if fields[i] == "mtu" { + mtu, err := strconv.Atoi(fields[i+1]) + if err != nil { + return 0, fmt.Errorf("failed to convert MTU to int: %v", err) + } + return mtu, nil + } + } + return 0, fmt.Errorf("failed to get MTU from stdout: %s", stdout) +} + func (data *TestData) GetNodeMACAddress(node, device string) (string, error) { antreaPod, err := data.getAntreaPodOnNode(node) if err != nil { diff --git a/test/e2e/mtu_test.go b/test/e2e/mtu_test.go new file mode 100644 index 00000000000..602bfee9c38 --- /dev/null +++ b/test/e2e/mtu_test.go @@ -0,0 +1,211 @@ +// Copyright 2024 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "testing" + + agentconfig "antrea.io/antrea/pkg/config/agent" +) + +func TestPodMTU(t *testing.T) { + data, err := setupTest(t) + if err != nil { + t.Fatalf("Error when setting up test: %v", err) + } + defer teardownTest(t, data) + + t.Run("testPodMTUWithGeneveTunnel", func(t *testing.T) { + testPodMTUWithGeneveTunnel(t, data) + }) + t.Run("testPodMTUWithVXLANTunnel", func(t *testing.T) { + testPodMTUWithVXLANTunnel(t, data) + }) + t.Run("testPodMTUWithGRETunnel", func(t *testing.T) { + testPodMTUWithGRETunnel(t, data) + }) + t.Run("testPodMTUWithWireGuard", func(t *testing.T) { + testPodMTUWithWireGuard(t, data) + }) +} + +func testPodMTUWithGeneveTunnel(t *testing.T, data *TestData) { + var previousTunnelType string + ac := func(config *agentconfig.AgentConfig) { + previousTunnelType = config.TunnelType + config.TunnelType = "geneve" + } + if err := data.mutateAntreaConfigMap(nil, ac, false, true); err != nil { + t.Fatalf("Failed to enable geneve tunnel: %v", err) + } + defer func() { + ac := func(config *agentconfig.AgentConfig) { + config.TunnelType = previousTunnelType + } + if err := data.mutateAntreaConfigMap(nil, ac, false, true); err != nil { + t.Errorf("Failed to enable %s tunnel: %v", previousTunnelType, err) + } + }() + + podName := "geneve" + err := data.createBusyboxPodOnNode(podName, data.testNamespace, workerNodeName(1), false) + if err != nil { + t.Fatalf("Failed to create busybox pod: %v", err) + } + defer deletePodWrapper(t, data, data.testNamespace, podName) + if err := data.podWaitForRunning(defaultTimeout, podName, data.testNamespace); err != nil { + t.Fatalf("Error when waiting for Pod %s: %v", podName, err) + } + + podInterface, err := data.GetPodInterfaceName(workerNodeName(1), podName) + if err != nil { + t.Fatalf("Failed to get Pod interface: %v", err) + } + mtu, err := data.GetInterfaceMTU(workerNodeName(1), podInterface) + if err != nil { + t.Fatalf("Failed to get interface MTU: %v", err) + } + if mtu != 1450 { + t.Errorf("Interface MTU is not 1450: %d", mtu) + } +} + +func testPodMTUWithVXLANTunnel(t *testing.T, data *TestData) { + var previousTunnelType string + ac := func(config *agentconfig.AgentConfig) { + previousTunnelType = config.TunnelType + config.TunnelType = "vxlan" + } + if err := data.mutateAntreaConfigMap(nil, ac, false, true); err != nil { + t.Fatalf("Failed to enable vxlan tunnel: %v", err) + } + defer func() { + ac := func(config *agentconfig.AgentConfig) { + config.TunnelType = previousTunnelType + } + if err := data.mutateAntreaConfigMap(nil, ac, false, true); err != nil { + t.Errorf("Failed to enable %s tunnel: %v", previousTunnelType, err) + } + }() + + podName := "vxlan" + err := data.createBusyboxPodOnNode(podName, data.testNamespace, workerNodeName(1), false) + if err != nil { + t.Fatalf("Failed to create busybox pod: %v", err) + } + defer deletePodWrapper(t, data, data.testNamespace, podName) + if err := data.podWaitForRunning(defaultTimeout, podName, data.testNamespace); err != nil { + t.Fatalf("Error when waiting for Pod %s: %v", podName, err) + } + + podInterface, err := data.GetPodInterfaceName(workerNodeName(1), podName) + if err != nil { + t.Fatalf("Failed to get Pod interface: %v", err) + } + mtu, err := data.GetInterfaceMTU(workerNodeName(1), podInterface) + if err != nil { + t.Fatalf("Failed to get interface MTU: %v", err) + } + if mtu != 1450 { + t.Errorf("Interface MTU is not 1450: %d", mtu) + } +} + +func testPodMTUWithGRETunnel(t *testing.T, data *TestData) { + skipIfIPv6Cluster(t) + var previousTunnelType string + ac := func(config *agentconfig.AgentConfig) { + previousTunnelType = config.TunnelType + config.TunnelType = "gre" + } + if err := data.mutateAntreaConfigMap(nil, ac, false, true); err != nil { + t.Fatalf("Failed to enable gre tunnel: %v", err) + } + defer func() { + ac := func(config *agentconfig.AgentConfig) { + config.TunnelType = previousTunnelType + } + if err := data.mutateAntreaConfigMap(nil, ac, false, true); err != nil { + t.Errorf("Failed to enable %s tunnel: %v", previousTunnelType, err) + } + }() + + podName := "grepod" + err := data.createBusyboxPodOnNode(podName, data.testNamespace, workerNodeName(1), false) + if err != nil { + t.Fatalf("Failed to create busybox pod: %v", err) + } + defer deletePodWrapper(t, data, data.testNamespace, podName) + if err := data.podWaitForRunning(defaultTimeout, podName, data.testNamespace); err != nil { + t.Fatalf("Error when waiting for Pod %s: %v", podName, err) + } + + podInterface, err := data.GetPodInterfaceName(workerNodeName(1), podName) + if err != nil { + t.Fatalf("Failed to get Pod interface: %v", err) + } + mtu, err := data.GetInterfaceMTU(workerNodeName(1), podInterface) + if err != nil { + t.Fatalf("Failed to get interface MTU: %v", err) + } + if mtu != 1462 { + t.Errorf("Interface MTU is not 1462: %d", mtu) + } +} + +func testPodMTUWithWireGuard(t *testing.T, data *TestData) { + var previousTunnelType string + var previousTrafficEncryptionMode string + ac := func(config *agentconfig.AgentConfig) { + previousTunnelType = config.TunnelType + previousTrafficEncryptionMode = config.TrafficEncryptionMode + config.TrafficEncryptionMode = "wireguard" + } + if err := data.mutateAntreaConfigMap(nil, ac, false, true); err != nil { + t.Fatalf("Failed to enable wireguard tunnel: %v", err) + } + defer func() { + ac := func(config *agentconfig.AgentConfig) { + config.TunnelType = previousTunnelType + config.TrafficEncryptionMode = previousTrafficEncryptionMode + } + if err := data.mutateAntreaConfigMap(nil, ac, false, true); err != nil { + t.Errorf("Failed to enable %s tunnel: %v", previousTunnelType, err) + } + }() + + podName := "wgpod" + err := data.createBusyboxPodOnNode(podName, data.testNamespace, workerNodeName(1), false) + if err != nil { + t.Fatalf("Failed to create busybox pod: %v", err) + } + defer deletePodWrapper(t, data, data.testNamespace, podName) + + if err := data.podWaitForRunning(defaultTimeout, podName, data.testNamespace); err != nil { + t.Fatalf("Error when waiting for Pod %s: %v", podName, err) + } + + podInterface, err := data.GetPodInterfaceName(workerNodeName(1), podName) + if err != nil { + t.Fatalf("Failed to get Pod interface: %v", err) + } + mtu, err := data.GetInterfaceMTU(workerNodeName(1), podInterface) + if err != nil { + t.Fatalf("Failed to get interface MTU: %v", err) + } + if mtu != 1420 { + t.Errorf("Interface MTU is not 1420: %d", mtu) + } +}