From 4310b5e5c4a15049d963cbad8791226567743e9c Mon Sep 17 00:00:00 2001 From: Lorenz Bauer Date: Thu, 9 Mar 2023 10:43:44 +0000 Subject: [PATCH] contrib/kind: enable XDP_TX from pod veth Both veth in a pair require an XDP program installed for XDP_TX to work. Since the host side veth created by kind doesn't have an XDP program attached we can't run any tests in CI that require XDP_TX. The workaround itself is just an ip link set and ethtool away, the problem is figuring out which interfaces we need to do the magic to. Use the approach used by kind-network-plugins and create our own docker network with a specific name for the bridge device. We can then iterate all children of the bridge and do our fixups. We tell kind to use our own network by setting the (undocumented?) KIND_EXPERIMENTAL_DOCKER_NETWORK environment variable. See https://github.com/aojea/kind-networking-plugins Signed-off-by: Lorenz Bauer --- .../workflows/conformance-datapath-v1.13.yaml | 6 +-- .github/workflows/conformance-datapath.yaml | 4 +- .gitignore | 3 ++ contrib/scripts/kind-down.sh | 8 ++- contrib/scripts/kind.sh | 46 +++++++++++++++++- test/l4lb/bpf_xdp_veth_host.o | Bin 0 -> 720 bytes 6 files changed, 58 insertions(+), 9 deletions(-) create mode 100644 test/l4lb/bpf_xdp_veth_host.o diff --git a/.github/workflows/conformance-datapath-v1.13.yaml b/.github/workflows/conformance-datapath-v1.13.yaml index 091059dcee03d..89875bd62cc61 100644 --- a/.github/workflows/conformance-datapath-v1.13.yaml +++ b/.github/workflows/conformance-datapath-v1.13.yaml @@ -286,7 +286,7 @@ jobs: provision: 'false' cmd: | cd /host/ - ./contrib/scripts/kind.sh "" 3 "" "" "${{ matrix.kube-proxy }}" "dual" + ./contrib/scripts/kind.sh --xdp "" 3 "" "" "${{ matrix.kube-proxy }}" "dual" ./cilium-cli install ${{ steps.vars.outputs.cilium_install_defaults }} ./cilium-cli status --wait @@ -294,7 +294,7 @@ jobs: --sysdump-output-filename "cilium-sysdump-${{ matrix.name }}-" ./cilium-cli connectivity test --collect-sysdump-on-failure \ --sysdump-output-filename "cilium-sysdump-${{ matrix.name }}-" - kind delete cluster + ./contrib/scripts/kind-down.sh --keep-registry - name: Run encryption tests if: ${{ matrix.encryption != 'disabled' }} @@ -303,7 +303,7 @@ jobs: provision: 'false' cmd: | cd /host/ - ./contrib/scripts/kind.sh "" 3 "" "" "${{ matrix.kube-proxy }}" "dual" + ./contrib/scripts/kind.sh --xdp "" 3 "" "" "${{ matrix.kube-proxy }}" "dual" ./cilium-cli install ${{ steps.vars.outputs.cilium_install_defaults }} \ --encryption=${{ matrix.encryption }} diff --git a/.github/workflows/conformance-datapath.yaml b/.github/workflows/conformance-datapath.yaml index 47e06d3bf5a76..b9995e2fbe177 100644 --- a/.github/workflows/conformance-datapath.yaml +++ b/.github/workflows/conformance-datapath.yaml @@ -386,7 +386,7 @@ jobs: provision: 'false' cmd: | cd /host/ - ./contrib/scripts/kind.sh "" 3 "" "" "${{ matrix.kube-proxy }}" "dual" + ./contrib/scripts/kind.sh --xdp "" 3 "" "" "${{ matrix.kube-proxy }}" "dual" ./cilium-cli install ${{ steps.vars.outputs.cilium_install_defaults }} ./cilium-cli status --wait @@ -396,7 +396,7 @@ jobs: --sysdump-output-filename "cilium-sysdump-${{ matrix.name }}-" ./cilium-cli connectivity test --collect-sysdump-on-failure \ --sysdump-output-filename "cilium-sysdump-${{ matrix.name }}-" - kind delete cluster + ./contrib/scripts/kind-down.sh --keep-registry - name: Fetch artifacts if: ${{ !success() }} diff --git a/.gitignore b/.gitignore index 7495d12f6e5c2..6d3eca312e09e 100644 --- a/.gitignore +++ b/.gitignore @@ -115,3 +115,6 @@ images/*/Dockerfile.dockerignore # Clangd cache for indexed bpf code bpf/.cache + +# Include dummy bpf object necessary for XDP_TX +!test/l4lb/bpf_xdp_veth_host.o diff --git a/contrib/scripts/kind-down.sh b/contrib/scripts/kind-down.sh index 1c025faa51231..0067262534974 100755 --- a/contrib/scripts/kind-down.sh +++ b/contrib/scripts/kind-down.sh @@ -9,6 +9,10 @@ if ! have_kind; then echo " https://kind.sigs.k8s.io/docs/user/quick-start/#installation" fi +if [ "${1:-}" != "--keep-registry" ]; then + docker kill kind-registry && \ + docker rm kind-registry +fi + kind delete clusters kind && \ -docker kill kind-registry && \ -docker rm kind-registry + docker network rm kind-cilium diff --git a/contrib/scripts/kind.sh b/contrib/scripts/kind.sh index 3150f99a8b12e..182eb14943431 100755 --- a/contrib/scripts/kind.sh +++ b/contrib/scripts/kind.sh @@ -14,8 +14,17 @@ default_pod_subnet="" default_service_subnet="" default_agent_port_prefix="234" default_operator_port_prefix="235" +default_network="kind-cilium" PROG=${0} + +xdp=false +if [ "${1:-}" = "--xdp" ]; then + xdp=true + shift +fi +readonly xdp + controlplanes="${1:-${CONTROLPLANES:=${default_controlplanes}}}" workers="${2:-${WORKERS:=${default_workers}}}" cluster_name="${3:-${CLUSTER_NAME:=${default_cluster_name}}}" @@ -27,10 +36,13 @@ pod_subnet="${PODSUBNET:=${default_pod_subnet}}" service_subnet="${SERVICESUBNET:=${default_service_subnet}}" agent_port_prefix="${AGENTPORTPREFIX:=${default_agent_port_prefix}}" operator_port_prefix="${OPERATORPORTPREFIX:=${default_operator_port_prefix}}" + +bridge_dev="br-${default_network}" +v6_prefix="fc00:c111::/64" CILIUM_ROOT="$(git rev-parse --show-toplevel)" usage() { - echo "Usage: ${PROG} [control-plane node count] [worker node count] [cluster-name] [node image] [kube-proxy mode] [ip-family]" + echo "Usage: ${PROG} [--xdp] [control-plane node count] [worker node count] [cluster-name] [node image] [kube-proxy mode] [ip-family]" } have_kind() { @@ -117,6 +129,16 @@ workers() { done } +# create a custom network so we can control the name of the bridge device. +# Inspired by https://github.com/kubernetes-sigs/kind/blob/6b58c9dfcbdb1b3a0d48754d043d59ca7073589b/pkg/cluster/internal/providers/docker/network.go#L149-L161 +docker network create -d=bridge \ + -o "com.docker.network.bridge.enable_ip_masquerade=true" \ + -o "com.docker.network.bridge.name=${bridge_dev}" \ + --ipv6 --subnet "${v6_prefix}" \ + "${default_network}" + +export KIND_EXPERIMENTAL_DOCKER_NETWORK="${default_network}" + # create a cluster with the local registry enabled in containerd cat < /dev/null + clang -O2 -Wall -target bpf -c bpf_xdp_veth_host.c -o bpf_xdp_veth_host.o + popd > /dev/null + fi + + for ifc in /sys/class/net/"${bridge_dev}"/brif/*; do + ifc="${ifc#"/sys/class/net/${bridge_dev}/brif/"}" + + # Attach a dummy XDP prog to the host side of the veth so that XDP_TX in the + # pod side works. + sudo ip link set dev "${ifc}" xdp obj "${CILIUM_ROOT}/test/l4lb/bpf_xdp_veth_host.o" + + # Disable TX and RX csum offloading, as veth does not support it. Otherwise, + # the forwarded packets by the LB to the worker node will have invalid csums. + sudo ethtool -K "${ifc}" rx off tx off > /dev/null + done +fi + +docker network connect "${default_network}" "${reg_name}" || true for node in $(kubectl get nodes --no-headers -o custom-columns=:.metadata.name); do kubectl annotate node "${node}" "kind.x-k8s.io/registry=localhost:${reg_port}"; diff --git a/test/l4lb/bpf_xdp_veth_host.o b/test/l4lb/bpf_xdp_veth_host.o new file mode 100644 index 0000000000000000000000000000000000000000..243879d135b56264ab3d275a80c50b34181e1f38 GIT binary patch literal 720 zcmb_ayKcfj5FFbm&;_CtsUu3{Z*Y@9;es;V1+aNYCTH0v1Qq#-d`lw!ii(o#S-Unm zAu3in&+OdndG~btxA@)nJVwcKU)O4y?c31NPK(4<>~ZIY|G*2oLJK$vBlwVEVJQLYwHDqGFcuQJ6npDSzJ zq-Lp>x-{VoVQC68nG){S7Vi@g9n(zFoOxh6ZlsD`Tx*?M?ZNRyPA;k3hvbh$Zq>`Z zioYX-vCsAH4AQszKezv9st@9`CitdR#~(BMR&2<8ZvBY%#8Vmh^TGJZXzQ#``hQ{l E9}%4@>Hq)$ literal 0 HcmV?d00001