Skip to content

Commit

Permalink
Add support for IPv6 pod range.
Browse files Browse the repository at this point in the history
Signed-off-by: Marek Chodor <[email protected]>
Change-Id: I93fe2040852e163f8ff1bdf8cbeb8d1c0c89f829
  • Loading branch information
marqc committed Feb 8, 2024
1 parent 2c8c4a3 commit 3911c6b
Show file tree
Hide file tree
Showing 7 changed files with 479 additions and 55 deletions.
215 changes: 160 additions & 55 deletions scripts/install-cni.sh
Original file line number Diff line number Diff line change
Expand Up @@ -130,20 +130,145 @@ else
cni_spec=${cni_spec//@cniIstioPlugin/}
fi

POPULATE_IP6TABLES="false"

# Fill CNI spec template.
ipv4_subnet=$(jq '.spec.podCIDR' <<<"$response")
#######################################
# Checks if given subnet is valid IPv4 range.
# Arguments:
# Subnet
# Returns:
# 0 if valid, 1 on invalid.
#######################################
function is_ipv4_range {
local IPV4_VALIDATION_REGEXP='^([0-9]{1,3}\.){3}[0-9]{1,3}\/[0-9]{1,2}$'
local ip=$1

[[ "${ip:-}" =~ ${IPV4_VALIDATION_REGEXP} ]]
}

if [[ "${ipv4_subnet:-}" =~ ^\"[0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*/[0-9][0-9]*\"$ ]]; then
echo "PodCIDR validation succeeded: ${ipv4_subnet:-}"
else
echo "Response from $node_url"
echo "$response"
echo "Failed to fetch/validate PodCIDR from K8s API server, ipv4_subnet=${ipv4_subnet:-}. Exiting (1)..."
exit 1
#######################################
# Checks if given subnet is valid IPv6 range.
# Arguments:
# Subnet
# Returns:
# 0 if valid, 1 on invalid.
#######################################
function is_ipv6_range {
local IPV6_VALIDATION_REGEXP='^(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))\/[0-9]{1,3}$'
local ip=$1

[[ "${ip:-}" =~ ${IPV6_VALIDATION_REGEXP} ]]
}

#######################################
# Replaces `@subnets` and `@routes` placeholders using `.spec.podCIDRs` from node json.
# In directpath use case it additionally adds IPv6 subnet derived from node's IPv6 address.
# Arguments:
# node json from kube-apiserver
# node_ipv6_addr node's IPv6 address from GCE metadata server
#######################################
function fillSubnetsInCniSpec {
local node=$1
local node_ipv6_addr=$2

local SUBNETS_REPLACEMENT=""
local ROUTES_REPLACEMENT=""

for subnet in $(jq -r '.spec.podCIDRs[]' <<<"$node") ; do
if is_ipv4_range "$subnet" ; then
if [ "" != "${SUBNETS_REPLACEMENT}" ] ; then
SUBNETS_REPLACEMENT="${SUBNETS_REPLACEMENT}, "
ROUTES_REPLACEMENT="${ROUTES_REPLACEMENT}, "
fi
SUBNETS_REPLACEMENT="${SUBNETS_REPLACEMENT}[{\"subnet\": \"${subnet}\"}]"
ROUTES_REPLACEMENT="${ROUTES_REPLACEMENT}{\"dst\": \"0.0.0.0/0\"}"
elif is_ipv6_range "$subnet" ; then
POPULATE_IP6TABLES="true"
if [ "" != "${SUBNETS_REPLACEMENT}" ] ; then
SUBNETS_REPLACEMENT="${SUBNETS_REPLACEMENT}, "
ROUTES_REPLACEMENT="${ROUTES_REPLACEMENT}, "
fi
SUBNETS_REPLACEMENT="${SUBNETS_REPLACEMENT}[{\"subnet\": \"${subnet}\"}]"
ROUTES_REPLACEMENT="${ROUTES_REPLACEMENT}{\"dst\": \"::/0\"}"
else
echo "[ERROR] Subnet $subnet is not a valid IP range"
exit 1
fi
done

# Directpath use case
if [ "$ENABLE_IPV6" == "true" ] ; then
# Directpath adds IPv6 subnet and route derived from host with fixed range
# of /112 even when it is not specified in node's .spec.podCIDRs
if [ -n "${node_ipv6_addr:-}" ] && [ "${node_ipv6_addr}" != "null" ]; then
POPULATE_IP6TABLES="true"
SUBNETS_REPLACEMENT="${SUBNETS_REPLACEMENT}, [{\"subnet\": \"${node_ipv6_addr}/112\"}]"
ROUTES_REPLACEMENT="${ROUTES_REPLACEMENT}, ${CNI_SPEC_IPV6_ROUTE:-{\"dst\": \"::/0\"\}}"
fi
fi

cni_spec=${cni_spec//@subnets/$SUBNETS_REPLACEMENT}
cni_spec=${cni_spec//@routes/$ROUTES_REPLACEMENT}
}

#######################################
# Replaces `@ipv4Subnet', '@ipv6SubnetOptional` and `@ipv6RouteOptional` placeholders using `.spec.podCIDR` from node json and node's ipv6 from metadata server.
# Arguments:
# node json from kube-apiserver
# node_ipv6_addr node's IPv6 address from GCE metadata server
#######################################
function fillSubnetsInCniSpecLegacyTemplate {
local node=$1
local node_ipv6_addr=$2

local primary_subnet
primary_subnet=$(jq -r '.spec.podCIDR' <<<"$response")

if is_ipv4_range "${primary_subnet:-}" ; then
echo "PodCIDR validation succeeded: ${primary_subnet:-}"
echo "Filling IPv4 subnet ${primary_subnet:-}"
cni_spec=${cni_spec//@ipv4Subnet/[{\"subnet\": \"${primary_subnet:-}\"\}]}
elif is_ipv6_range "${primary_subnet:-}" ; then
echo "Primary IPv6 pod range detected '${primary_subnet:-}'. It will only work with new spec template."
else
echo "Response from $node_url"
echo "$response"
echo "Failed to fetch/validate PodCIDR from K8s API server, primary_subnet=${primary_subnet:-}. Exiting (1)..."
exit 1
fi

if [ "$ENABLE_IPV6" == "true" ] || [ "${STACK_TYPE:-}" == "IPV4_IPV6" ]; then
if [ -n "${node_ipv6_addr:-}" ] && [ "${node_ipv6_addr}" != "null" ]; then
echo "Found nic0 IPv6 address ${node_ipv6_addr:-}. Filling IPv6 subnet and route..."
POPULATE_IP6TABLES="true"

cni_spec=${cni_spec//@ipv6SubnetOptional/, [{\"subnet\": \"${node_ipv6_addr:-}/112\"\}]}
cni_spec=${cni_spec//@ipv6RouteOptional/, ${CNI_SPEC_IPV6_ROUTE:-{\"dst\": \"::/0\"\}}}
else
echo "No IPv6 address found for nic0. Clearing IPv6 subnet and route..."
cni_spec=${cni_spec//@ipv6SubnetOptional/}
cni_spec=${cni_spec//@ipv6RouteOptional/}
fi
else
echo "Clearing IPv6 subnet and route given IPv6 access is disabled..."
cni_spec=${cni_spec//@ipv6SubnetOptional/}
cni_spec=${cni_spec//@ipv6RouteOptional/}
fi

}

STACK_TYPE=$(jq -r '.metadata.labels."cloud.google.com/gke-stack-type"' <<<"$response")
echo "Node stack type label: '${STACK_TYPE:-}'"

node_ipv6_addr=''
if [ "$ENABLE_IPV6" == "true" ] || [ "${STACK_TYPE:-}" == "IPV4_IPV6" ]; then
node_ipv6_addr=$(curl -s -k --fail "http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/0/?recursive=true" -H "Metadata-Flavor: Google" | jq -r '.ipv6s[0]' ) ||:
fi

echo "Filling IPv4 subnet ${ipv4_subnet:-}"
cni_spec=${cni_spec//@ipv4Subnet/[{\"subnet\": ${ipv4_subnet:-}\}]}
fillSubnetsInCniSpec "$response" "$node_ipv6_addr"
fillSubnetsInCniSpecLegacyTemplate "$response" "$node_ipv6_addr"


if [ "$ENABLE_MASQUERADE" == "true" ]; then
echo "Config MASQUERADE rule"
Expand All @@ -170,56 +295,36 @@ if [ "$ENABLE_MASQUERADE" == "true" ]; then
fi
fi

STACK_TYPE=$(jq '.metadata.labels."cloud.google.com/gke-stack-type"' <<<"$response")
echo "Node stack type label: '${STACK_TYPE:-}'"

if [ "$ENABLE_IPV6" == "true" ] || [ "${STACK_TYPE:-}" == '"IPV4_IPV6"' ]; then
node_ipv6_addr=$(curl -s -k --fail "http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/0/?recursive=true" -H "Metadata-Flavor: Google" | jq -r '.ipv6s[0]' ) ||:

if [ -n "${node_ipv6_addr:-}" ] && [ "${node_ipv6_addr}" != "null" ]; then
echo "Found nic0 IPv6 address ${node_ipv6_addr:-}. Filling IPv6 subnet and route..."

cni_spec=${cni_spec//@ipv6SubnetOptional/, [{\"subnet\": \"${node_ipv6_addr:-}/112\"\}]}
cni_spec=${cni_spec//@ipv6RouteOptional/, ${CNI_SPEC_IPV6_ROUTE:-{\"dst\": \"::/0\"\}}}
if [ "$POPULATE_IP6TABLES" == "true" ] ; then
# Ensure the IPv6 firewall rules are as expected.
# These rules mirror the IPv4 rules installed by kubernetes/cluster/gce/gci/configure-helper.sh

# Ensure the IPv6 firewall rules are as expected.
# These rules mirror the IPv4 rules installed by kubernetes/cluster/gce/gci/configure-helper.sh

if ip6tables -w -L INPUT | grep "Chain INPUT (policy DROP)" > /dev/null; then
echo "Add rules to accept all inbound TCP/UDP/ICMP/SCTP IPv6 packets"
ip6tables -A INPUT -w -p tcp -j ACCEPT
ip6tables -A INPUT -w -p udp -j ACCEPT
ip6tables -A INPUT -w -p icmpv6 -j ACCEPT
ip6tables -A INPUT -w -p sctp -j ACCEPT
fi
if ip6tables -w -L INPUT | grep "Chain INPUT (policy DROP)" > /dev/null; then
echo "Add rules to accept all inbound TCP/UDP/ICMP/SCTP IPv6 packets"
ip6tables -A INPUT -w -p tcp -j ACCEPT
ip6tables -A INPUT -w -p udp -j ACCEPT
ip6tables -A INPUT -w -p icmpv6 -j ACCEPT
ip6tables -A INPUT -w -p sctp -j ACCEPT
fi

if ip6tables -w -L FORWARD | grep "Chain FORWARD (policy DROP)" > /dev/null; then
echo "Add rules to accept all forwarded TCP/UDP/ICMP/SCTP IPv6 packets"
ip6tables -A FORWARD -w -p tcp -j ACCEPT
ip6tables -A FORWARD -w -p udp -j ACCEPT
ip6tables -A FORWARD -w -p icmpv6 -j ACCEPT
ip6tables -A FORWARD -w -p sctp -j ACCEPT
fi
if ip6tables -w -L FORWARD | grep "Chain FORWARD (policy DROP)" > /dev/null; then
echo "Add rules to accept all forwarded TCP/UDP/ICMP/SCTP IPv6 packets"
ip6tables -A FORWARD -w -p tcp -j ACCEPT
ip6tables -A FORWARD -w -p udp -j ACCEPT
ip6tables -A FORWARD -w -p icmpv6 -j ACCEPT
ip6tables -A FORWARD -w -p sctp -j ACCEPT
fi

# Ensure the other IPv6 rules we need are also installed, and in before any other node rules.
# Always allow ICMP
ip6tables -I OUTPUT -p icmpv6 -j ACCEPT -w
# Accept new and return traffic outbound
ip6tables -I OUTPUT -m state --state NEW,ESTABLISHED,RELATED -j ACCEPT -w
# Ensure the other IPv6 rules we need are also installed, and in before any other node rules.
# Always allow ICMP
ip6tables -I OUTPUT -p icmpv6 -j ACCEPT -w
# Accept new and return traffic outbound
ip6tables -I OUTPUT -m state --state NEW,ESTABLISHED,RELATED -j ACCEPT -w

if [ "${ENABLE_CALICO_NETWORK_POLICY}" == "true" ]; then
echo "Enabling IPv6 forwarding..."
echo 1 > /proc/sys/net/ipv6/conf/all/forwarding
fi
else
echo "No IPv6 address found for nic0. Clearing IPv6 subnet and route..."
cni_spec=${cni_spec//@ipv6SubnetOptional/}
cni_spec=${cni_spec//@ipv6RouteOptional/}
if [ "${ENABLE_CALICO_NETWORK_POLICY}" == "true" ]; then
echo "Enabling IPv6 forwarding..."
echo 1 > /proc/sys/net/ipv6/conf/all/forwarding
fi
else
echo "Clearing IPv6 subnet and route given IPv6 access is disabled..."
cni_spec=${cni_spec//@ipv6SubnetOptional/}
cni_spec=${cni_spec//@ipv6RouteOptional/}
fi

# MTU to use if the interface in use can't be detected.
Expand Down
64 changes: 64 additions & 0 deletions scripts/testcase/testcase-basic-v2.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
export KUBERNETES_SERVICE_HOST=kubernetes.default.svc
export KUBERNETES_SERVICE_PORT=443

export ENABLE_CALICO_NETWORK_POLICY=false
export ENABLE_BANDWIDTH_PLUGIN=false
export ENABLE_CILIUM_PLUGIN=false
export ENABLE_MASQUERADE=false
export ENABLE_IPV6=false

CNI_SPEC_TEMPLATE=$(cat testdata/spec-template-v2.json)
export CNI_SPEC_TEMPLATE

function before_test() {

function curl() {
# shellcheck disable=SC2317
case "$*" in
*http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/0*)
echo '{"ipv6s": ["2600:1900:4000:318:0:7:0:0"]}'
;;
*https://kubernetes.default.svc:443/api/v1/nodes/*)
echo '{
"metadata": {
"labels": {
},
"creationTimestamp": "2024-01-03T11:54:01Z",
"name": "gke-my-cluster-default-pool-128bc25d-9c94",
"resourceVersion": "891003",
"uid": "f2353a2f-ca8c-4ca0-8dd3-ad1f964a54f0"
},
"spec": {
"podCIDR": "10.52.1.0/24",
"podCIDRs": [
"10.52.1.0/24"
],
"providerID": "gce://my-gke-project/us-central1-c/gke-my-cluster-default-pool-128bc25d-9c94"
}
}'
;;
*)
#unsupported
exit 1
esac
}
export -f curl

}

function verify() {
local expected
local actual

expected=$(jq -S . <"testdata/expected-basic.json")
actual=$(jq -S . <"/host/etc/cni/net.d/${CNI_SPEC_NAME}")

if [ "$expected" != "$actual" ] ; then
echo "Expected cni_spec value:"
echo "$expected"
echo "but actual was"
echo "$actual"
return 1
fi

}
68 changes: 68 additions & 0 deletions scripts/testcase/testcase-directpath-v2.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
export KUBERNETES_SERVICE_HOST=kubernetes.default.svc
export KUBERNETES_SERVICE_PORT=443

export ENABLE_CALICO_NETWORK_POLICY=false
export ENABLE_BANDWIDTH_PLUGIN=false
export ENABLE_CILIUM_PLUGIN=false
export ENABLE_MASQUERADE=false
export ENABLE_IPV6=true
export CNI_SPEC_IPV6_ROUTE="{\"dst\": \"2600:1900:4000::/42\"}"


CNI_SPEC_TEMPLATE=$(cat testdata/spec-template-v2.json)
export CNI_SPEC_TEMPLATE

function before_test() {

function curl() {
# shellcheck disable=SC2317
case "$*" in
*http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/0*)
# call to GCE metadata server
echo '{"ipv6s": ["2600:1900:4000:318:0:7:0:0"]}'
;;
*https://kubernetes.default.svc:443/api/v1/nodes/*)
# call to kube-apiserver
echo '{
"metadata": {
"labels": {
},
"creationTimestamp": "2024-01-03T11:54:01Z",
"name": "gke-my-cluster-default-pool-128bc25d-9c94",
"resourceVersion": "891003",
"uid": "f2353a2f-ca8c-4ca0-8dd3-ad1f964a54f0"
},
"spec": {
"podCIDR": "10.52.1.0/24",
"podCIDRs": [
"10.52.1.0/24"
],
"providerID": "gce://my-gke-project/us-central1-c/gke-my-cluster-default-pool-128bc25d-9c94"
}
}'
;;
*)
# unmatched call
exit 1
esac
}
export -f curl

}

function verify() {
local expected
local actual

expected=$(jq -S . <"testdata/expected-directpath.json")
actual=$(jq -S . <"/host/etc/cni/net.d/${CNI_SPEC_NAME}")

if [ "$expected" != "$actual" ] ; then
echo "Expected cni_spec value:"
echo "$expected"
echo "but actual was"
echo "$actual"
return 1
fi

}
Loading

0 comments on commit 3911c6b

Please sign in to comment.