Skip to content

Commit

Permalink
Bugfix: Fix incorrect Pod MTU when WireGuard enabled
Browse files Browse the repository at this point in the history
The default tunnel type is GENEVE, when the WireGuard encryption
is enabled, the Pod MTU calculator still deducts the GENEVE overhead
directly, which causes a MTU error.

This patch will calculate all MTU deduction in function
CalculateMTUDeduction, including WireGuard and Multicluster.

Signed-off-by: Jiajing Hu <[email protected]>
  • Loading branch information
hjiajing committed Jan 22, 2024
1 parent 48d7f7b commit 0cbb17f
Show file tree
Hide file tree
Showing 7 changed files with 295 additions and 14 deletions.
2 changes: 1 addition & 1 deletion cmd/antrea-agent/agent.go
Original file line number Diff line number Diff line change
Expand Up @@ -205,7 +205,7 @@ func run(o *Options) error {
IPsecConfig: config.IPsecConfig{
AuthenticationMode: ipsecAuthenticationMode,
},
EnableMulticlusterGW: enableMulticlusterGW,
MulticlusterConfig: o.config.Multicluster,
}

wireguardConfig := &config.WireGuardConfig{
Expand Down
4 changes: 0 additions & 4 deletions pkg/agent/agent.go
Original file line number Diff line number Diff line change
Expand Up @@ -1194,10 +1194,6 @@ func (i *Initializer) getInterfaceMTU(transportInterface *net.Interface) (int, e

isIPv6 := i.nodeConfig.NodeIPv6Addr != nil
mtu -= i.networkConfig.CalculateMTUDeduction(isIPv6)

if i.networkConfig.TrafficEncryptionMode == config.TrafficEncryptionModeIPSec {
mtu -= config.IPSecESPOverhead
}
return mtu, nil
}

Expand Down
29 changes: 21 additions & 8 deletions pkg/agent/config/node_config.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,9 @@ package config
import (
"fmt"
"net"
"strings"

agentConfig "antrea.io/antrea/pkg/config/agent"
"antrea.io/antrea/pkg/ovs/ovsconfig"
)

Expand Down Expand Up @@ -201,14 +203,15 @@ type NetworkConfig struct {
TransportIfaceCIDRs []string
IPv4Enabled bool
IPv6Enabled bool
// MTUDeduction only counts IPv4 tunnel overhead, no IPsec and WireGuard overhead.
// MTUDeduction is the MTU deduction for encapsulation and encryption in cluster.
MTUDeduction int
// Set by the defaultMTU config option or auto discovered.
// Auto discovery will use MTU value of the Node's transport interface.
// For Encap and Hybrid mode, InterfaceMTU will be adjusted to account for
// encap header.
InterfaceMTU int
EnableMulticlusterGW bool
MulticlusterConfig agentConfig.MulticlusterConfig
}

// IsIPv4Enabled returns true if the cluster network supports IPv4. Legal cases are:
Expand Down Expand Up @@ -265,23 +268,33 @@ func (nc *NetworkConfig) NeedsDirectRoutingToPeer(peerIP net.IP, localIP *net.IP
}

func (nc *NetworkConfig) CalculateMTUDeduction(isIPv6 bool) int {
var mtuDeduction int
// When WireGuard is enabled, NetworkConfig.TunnelType will be ignored, so we deduct MTU based on WireGuardOverhead.
if nc.TrafficEncryptionMode == TrafficEncryptionModeWireGuard {
nc.MTUDeduction = WireGuardOverhead
return nc.MTUDeduction
} else if nc.TrafficEncryptionMode == TrafficEncryptionModeIPSec {
nc.MTUDeduction = IPSecESPOverhead
}

// When Multi-cluster Gateway is enabled, we need to reduce MTU for potential cross-cluster traffic.
if nc.TrafficEncapMode.SupportsEncap() || nc.EnableMulticlusterGW {
if nc.TunnelType == ovsconfig.VXLANTunnel {
mtuDeduction = vxlanOverhead
nc.MTUDeduction += vxlanOverhead
} else if nc.TunnelType == ovsconfig.GeneveTunnel {
mtuDeduction = geneveOverhead
nc.MTUDeduction += geneveOverhead
} else if nc.TunnelType == ovsconfig.GRETunnel {
mtuDeduction = greOverhead
nc.MTUDeduction += greOverhead
}
}

if nc.TrafficEncapMode.SupportsEncap() && isIPv6 {
mtuDeduction += ipv6ExtraOverhead
nc.MTUDeduction += ipv6ExtraOverhead
}
// When multi-cluster WireGuard is enabled, we need to reduce MTU for potential cross-cluster traffic.
if nc.EnableMulticlusterGW && strings.EqualFold(nc.MulticlusterConfig.TrafficEncryptionMode, TrafficEncryptionModeWireGuard.String()) {
nc.MTUDeduction += WireGuardOverhead
}
nc.MTUDeduction = mtuDeduction
return mtuDeduction
return nc.MTUDeduction
}

// ServiceConfig includes K8s Service CIDR and available IP addresses for NodePort.
Expand Down
20 changes: 20 additions & 0 deletions pkg/agent/config/node_config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (

"github.com/stretchr/testify/assert"

agentConfig "antrea.io/antrea/pkg/config/agent"
"antrea.io/antrea/pkg/ovs/ovsconfig"
)

Expand Down Expand Up @@ -306,6 +307,25 @@ func TestCalculateMTUDeduction(t *testing.T) {
isIPv6: true,
expectedMTUDeduction: 70,
},
{
name: "WireGuard enabled",
nc: &NetworkConfig{TrafficEncryptionMode: TrafficEncryptionModeWireGuard},
expectedMTUDeduction: 80,
},
{
name: "Multicluster enabled with Geneve encap",
nc: &NetworkConfig{TunnelType: ovsconfig.GeneveTunnel, EnableMulticlusterGW: true},
expectedMTUDeduction: 50,
},
{
name: "Geneve encap with Multicluster WireGuard enabled",
nc: &NetworkConfig{
TunnelType: ovsconfig.GeneveTunnel,
EnableMulticlusterGW: true,
MulticlusterConfig: agentConfig.MulticlusterConfig{TrafficEncryptionMode: "wireGuard"},
},
expectedMTUDeduction: 130,
},
}

for _, tt := range tests {
Expand Down
2 changes: 1 addition & 1 deletion pkg/agent/multicluster/mc_route_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,7 @@ func NewMCDefaultRouteController(
controller.wireGuardConfig = &config.WireGuardConfig{
Port: multiclusterConfig.WireGuard.Port,
Name: multiclusterWireGuardInterface,
MTU: controller.nodeConfig.NodeTransportInterfaceMTU - controller.networkConfig.MTUDeduction - config.WireGuardOverhead,
MTU: controller.nodeConfig.NodeTransportInterfaceMTU - controller.networkConfig.MTUDeduction,
}
}
controller.gwInformer.Informer().AddEventHandlerWithResyncPeriod(
Expand Down
41 changes: 41 additions & 0 deletions test/e2e/framework.go
Original file line number Diff line number Diff line change
Expand Up @@ -2481,6 +2481,47 @@ func (data *TestData) GetTransportInterface() (string, error) {
return "", fmt.Errorf("no interface was assigned with Node IP %s", nodeIP)
}

func (data *TestData) GetPodInterfaceName(nodeName string, podName string) (string, error) {
antreaPod, err := data.getAntreaPodOnNode(nodeName)
if err != nil {
return "", fmt.Errorf("failed to get Antrea Pod on Node %s: %v", nodeName, err)
}
cmd := []string{"ip", "-br", "addr", "show"}
stdout, stderr, err := data.RunCommandFromPod(antreaNamespace, antreaPod, agentContainerName, cmd)
if stdout == "" || stderr != "" || err != nil {
return "", fmt.Errorf("failed to show ip address, stdout: %s, stderr: %s, err: %v", stdout, stderr, err)
}

lines := strings.Split(strings.TrimSpace(stdout), "\n")
for _, line := range lines {
if strings.Contains(line, podName) {
fields := strings.Fields(line)
name, _, _ := strings.Cut(fields[0], "@")
return name, nil
}
}

return "", fmt.Errorf("no interface was assigned with Pod %s", podName)
}

func (data *TestData) GetInterfaceMTU(nodeName string, interfaceName string) (int, error) {
antreaPod, err := data.getAntreaPodOnNode(nodeName)
if err != nil {
return 0, fmt.Errorf("failed to get Antrea Pod on Node %s: %v", nodeName, err)
}
cmd := []string{"cat", "/sys/class/net/" + interfaceName + "/mtu"}
stdout, stderr, err := data.RunCommandFromPod(antreaNamespace, antreaPod, agentContainerName, cmd)
if stdout == "" || stderr != "" || err != nil {
return 0, fmt.Errorf("failed to get interface MTU, stdout: %s, stderr: %s, err: %v", stdout, stderr, err)
}

mtu, err := strconv.Atoi(strings.TrimSpace(stdout))
if err != nil {
return 0, fmt.Errorf("failed to convert MTU to int: %v", err)
}
return mtu, nil
}

func (data *TestData) GetNodeMACAddress(node, device string) (string, error) {
antreaPod, err := data.getAntreaPodOnNode(node)
if err != nil {
Expand Down
211 changes: 211 additions & 0 deletions test/e2e/mtu_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,211 @@
// Copyright 2024 Antrea Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package e2e

import (
"testing"

agentconfig "antrea.io/antrea/pkg/config/agent"
)

func TestPodMTU(t *testing.T) {
data, err := setupTest(t)
if err != nil {
t.Fatalf("Error when setting up test: %v", err)
}
defer teardownTest(t, data)

t.Run("testPodMTUWithGeneveTunnel", func(t *testing.T) {
testPodMTUWithGeneveTunnel(t, data)
})
t.Run("testPodMTUWithVXLANTunnel", func(t *testing.T) {
testPodMTUWithVXLANTunnel(t, data)
})
t.Run("testPodMTUWithGRETunnel", func(t *testing.T) {
testPodMTUWithGRETunnel(t, data)
})
t.Run("testPodMTUWithWireGuard", func(t *testing.T) {
testPodMTUWithWireGuard(t, data)
})
}

func testPodMTUWithGeneveTunnel(t *testing.T, data *TestData) {
var previousTunnelType string
ac := func(config *agentconfig.AgentConfig) {
previousTunnelType = config.TunnelType
config.TunnelType = "geneve"
}
if err := data.mutateAntreaConfigMap(nil, ac, false, true); err != nil {
t.Fatalf("Failed to enable geneve tunnel: %v", err)
}
defer func() {
ac := func(config *agentconfig.AgentConfig) {
config.TunnelType = previousTunnelType
}
if err := data.mutateAntreaConfigMap(nil, ac, false, true); err != nil {
t.Errorf("Failed to enable %s tunnel: %v", previousTunnelType, err)
}
}()

podName := "geneve"
err := data.createBusyboxPodOnNode(podName, data.testNamespace, workerNodeName(1), false)
if err != nil {
t.Fatalf("Failed to create busybox pod: %v", err)
}
defer deletePodWrapper(t, data, data.testNamespace, podName)
if err := data.podWaitForRunning(defaultTimeout, podName, data.testNamespace); err != nil {
t.Fatalf("Error when waiting for Pod %s: %v", podName, err)
}

podInterface, err := data.GetPodInterfaceName(workerNodeName(1), podName)
if err != nil {
t.Fatalf("Failed to get Pod interface: %v", err)
}
mtu, err := data.GetInterfaceMTU(workerNodeName(1), podInterface)
if err != nil {
t.Fatalf("Failed to get interface MTU: %v", err)
}
if mtu != 1450 {
t.Errorf("Interface MTU is not 1450: %d", mtu)
}
}

func testPodMTUWithVXLANTunnel(t *testing.T, data *TestData) {
var previousTunnelType string
ac := func(config *agentconfig.AgentConfig) {
previousTunnelType = config.TunnelType
config.TunnelType = "vxlan"
}
if err := data.mutateAntreaConfigMap(nil, ac, false, true); err != nil {
t.Fatalf("Failed to enable vxlan tunnel: %v", err)
}
defer func() {
ac := func(config *agentconfig.AgentConfig) {
config.TunnelType = previousTunnelType
}
if err := data.mutateAntreaConfigMap(nil, ac, false, true); err != nil {
t.Errorf("Failed to enable %s tunnel: %v", previousTunnelType, err)
}
}()

podName := "vxlanpod"
err := data.createBusyboxPodOnNode(podName, data.testNamespace, workerNodeName(1), false)
if err != nil {
t.Fatalf("Failed to create busybox pod: %v", err)
}
defer deletePodWrapper(t, data, data.testNamespace, podName)
if err := data.podWaitForRunning(defaultTimeout, podName, data.testNamespace); err != nil {
t.Fatalf("Error when waiting for Pod %s: %v", podName, err)
}

podInterface, err := data.GetPodInterfaceName(workerNodeName(1), podName)
if err != nil {
t.Fatalf("Failed to get Pod interface: %v", err)
}
mtu, err := data.GetInterfaceMTU(workerNodeName(1), podInterface)
if err != nil {
t.Fatalf("Failed to get interface MTU: %v", err)
}
if mtu != 1450 {
t.Errorf("Interface MTU is not 1450: %d", mtu)
}
}

func testPodMTUWithGRETunnel(t *testing.T, data *TestData) {
skipIfIPv6Cluster(t)
var previousTunnelType string
ac := func(config *agentconfig.AgentConfig) {
previousTunnelType = config.TunnelType
config.TunnelType = "gre"
}
if err := data.mutateAntreaConfigMap(nil, ac, false, true); err != nil {
t.Fatalf("Failed to enable gre tunnel: %v", err)
}
defer func() {
ac := func(config *agentconfig.AgentConfig) {
config.TunnelType = previousTunnelType
}
if err := data.mutateAntreaConfigMap(nil, ac, false, true); err != nil {
t.Errorf("Failed to enable %s tunnel: %v", previousTunnelType, err)
}
}()

podName := "grepod"
err := data.createBusyboxPodOnNode(podName, data.testNamespace, workerNodeName(1), false)
if err != nil {
t.Fatalf("Failed to create busybox pod: %v", err)
}
defer deletePodWrapper(t, data, data.testNamespace, podName)
if err := data.podWaitForRunning(defaultTimeout, podName, data.testNamespace); err != nil {
t.Fatalf("Error when waiting for Pod %s: %v", podName, err)
}

podInterface, err := data.GetPodInterfaceName(workerNodeName(1), podName)
if err != nil {
t.Fatalf("Failed to get Pod interface: %v", err)
}
mtu, err := data.GetInterfaceMTU(workerNodeName(1), podInterface)
if err != nil {
t.Fatalf("Failed to get interface MTU: %v", err)
}
if mtu != 1462 {
t.Errorf("Interface MTU is not 1462: %d", mtu)
}
}

func testPodMTUWithWireGuard(t *testing.T, data *TestData) {
var previousTunnelType string
var previousTrafficEncryptionMode string
ac := func(config *agentconfig.AgentConfig) {
previousTunnelType = config.TunnelType
previousTrafficEncryptionMode = config.TrafficEncryptionMode
config.TrafficEncryptionMode = "wireguard"
}
if err := data.mutateAntreaConfigMap(nil, ac, false, true); err != nil {
t.Fatalf("Failed to enable wireguard tunnel: %v", err)
}
defer func() {
ac := func(config *agentconfig.AgentConfig) {
config.TunnelType = previousTunnelType
config.TrafficEncryptionMode = previousTrafficEncryptionMode
}
if err := data.mutateAntreaConfigMap(nil, ac, false, true); err != nil {
t.Errorf("Failed to enable %s tunnel: %v", previousTunnelType, err)
}
}()

podName := "wgpod"
err := data.createBusyboxPodOnNode(podName, data.testNamespace, workerNodeName(1), false)
if err != nil {
t.Fatalf("Failed to create busybox pod: %v", err)
}
defer deletePodWrapper(t, data, data.testNamespace, podName)

if err := data.podWaitForRunning(defaultTimeout, podName, data.testNamespace); err != nil {
t.Fatalf("Error when waiting for Pod %s: %v", podName, err)
}

podInterface, err := data.GetPodInterfaceName(workerNodeName(1), podName)
if err != nil {
t.Fatalf("Failed to get Pod interface: %v", err)
}
mtu, err := data.GetInterfaceMTU(workerNodeName(1), podInterface)
if err != nil {
t.Fatalf("Failed to get interface MTU: %v", err)
}
if mtu != 1420 {
t.Errorf("Interface MTU is not 1420: %d", mtu)
}
}

0 comments on commit 0cbb17f

Please sign in to comment.