From 3f8c169ca618be690b70803ce0cf68679dc76f6d Mon Sep 17 00:00:00 2001 From: hjiajing Date: Wed, 25 Oct 2023 21:14:28 +0800 Subject: [PATCH] Add scrips for migrating from other CNI to Antrea Add scrtip and antctl subcommand "migrate" to migrate from other CNI(Calico, flannel) to Antrea. It also supports conditional NetworkPolicy conversion. Signed-off-by: hjiajing --- build/images/scripts/restart_sandbox | 61 +++ build/yamls/antrea-migrator.yml | 59 +++ docs/migrate-to-antrea.md | 85 ++++ go.mod | 5 +- go.sum | 9 +- hack/migrate-to-antrea.sh | 221 ++++++++++ pkg/antctl/antctl.go | 14 + pkg/antctl/command_definition.go | 6 + pkg/antctl/command_list.go | 3 +- pkg/antctl/raw/migrate/checkenv/command.go | 290 ++++++++++++ .../raw/migrate/checkenv/command_test.go | 61 +++ pkg/antctl/raw/migrate/common.go | 28 ++ .../migrate/convertnetworkpolicy/command.go | 414 ++++++++++++++++++ .../crd/v1alpha1/clusternetworkpolicy.go | 5 +- 14 files changed, 1253 insertions(+), 8 deletions(-) create mode 100755 build/images/scripts/restart_sandbox create mode 100644 build/yamls/antrea-migrator.yml create mode 100644 docs/migrate-to-antrea.md create mode 100755 hack/migrate-to-antrea.sh create mode 100644 pkg/antctl/raw/migrate/checkenv/command.go create mode 100644 pkg/antctl/raw/migrate/checkenv/command_test.go create mode 100644 pkg/antctl/raw/migrate/common.go create mode 100644 pkg/antctl/raw/migrate/convertnetworkpolicy/command.go diff --git a/build/images/scripts/restart_sandbox b/build/images/scripts/restart_sandbox new file mode 100755 index 00000000000..4c54a5cb9bf --- /dev/null +++ b/build/images/scripts/restart_sandbox @@ -0,0 +1,61 @@ +#!/usr/bin/env bash + +# Copyright 2022 Antrea Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +SANDBOX_ID_ANNOTATION="io.kubernetes.cri.sandbox-id" +CONTAINERD_BINARIES_URL="https://github.com/containerd/containerd/releases/download/v1.7.7/containerd-1.7.7-linux-amd64.tar.gz" + +# Remove rules of other CNI in CHAIN CNI-HOSTPORT-DNAT +iptables -t nat -F CNI-HOSTPORT-DNAT || true +chains=$(iptables -t nat -L | grep CNI-DN | grep -v "antrea" | awk '{print $2}') +for chain in $chains; do + iptables -t nat -X "$chain" || true +done + +wget $CONTAINERD_BINARIES_URL +tar xvf containerd-1.7.7-linux-amd64.tar.gz +mv ./bin/ctr /usr/local/bin/ctr + +pause_container_ids=$(ctr -n k8s.io containers ls | grep -v "CONTAINER" | grep "registry.k8s.io/pause" | awk '{print $1}') +# If the container's linux.namespaces fields contains "network", it means that the container uses CNI network. +# In order to switch CNI, we need to kill the corresponding tasks to restart the Pods. +for container_id in $pause_container_ids; do + container_info=$(ctr -n k8s.io containers info "$container_id" --spec) + sandbox_id=$(echo "$container_info" | jq -r .annotations.\"$SANDBOX_ID_ANNOTATION\") + namespaces=$(echo "$container_info" | jq .linux.namespaces | jq -c -r '.[]') + for namespace in $namespaces; do + if [[ "$(echo "$namespace" | jq .type)" == "\"network\"" ]]; then + echo "Container $container_id uses CNI network, kill the corresponding task" + ctr -n k8s.io tasks kill "$sandbox_id" || true + ctr -n k8s.io container remove "$container_id" || true + fi + done +done + +# After restart, some containers may be dangling without a pause container, remove them +tasks=$(ctr -n k8s.io tasks list -q) +container_ids=$(ctr -n k8s.io containers ls | grep -v "CONTAINER" | grep -v "registry.k8s.io/pause" | awk '{print $1}') +for container_id in $container_ids; do + sandbox_id=$(ctr -n k8s.io containers info "$container_id" --spec | jq -r .annotations.\"$SANDBOX_ID_ANNOTATION\") + if [[ ! "$tasks" =~ $sandbox_id ]]; then + echo "Container $container_id has no corresponding task, remove it" + ctr -n k8s.io tasks kill "$container_id" || true + fi +done + +rm /host/etc/cni/net.d/10-calico.conflist || true +rm /host/etc/cni/net.d/10-flannel.conflist || true diff --git a/build/yamls/antrea-migrator.yml b/build/yamls/antrea-migrator.yml new file mode 100644 index 00000000000..2e7459a43e0 --- /dev/null +++ b/build/yamls/antrea-migrator.yml @@ -0,0 +1,59 @@ +kind: DaemonSet +apiVersion: apps/v1 +metadata: + labels: + app: antrea + component: antrea-migrator + name: antrea-migrator + namespace: kube-system +spec: + selector: + matchLabels: + app: antrea + component: antrea-migrator + template: + metadata: + labels: + app: antrea + component: antrea-migrator + spec: + hostPID: true + hostNetwork: true + nodeSelector: + kubernetes.io/os: linux + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists + serviceAccountName: antrea-agent + volumes: + - name: containerd + hostPath: + path: /run/containerd + initContainers: + # initContainer will kill all sandboxes container + - name: antrea-migrator-init + image: antrea/antrea-ubuntu:latest + imagePullPolicy: IfNotPresent + securityContext: + privileged: true + capabilities: + add: + # SYS_MODULE is required to load the OVS kernel module. + - SYS_MODULE + command: + - restart_sandbox + volumeMounts: + - mountPath: /run/containerd + name: containerd + containers: + # If the all antrea-migrator Pods are running, it means that all initContainers have been completed + - image: antrea/antrea-ubuntu:latest + imagePullPolicy: IfNotPresent + name: antrea-migrator + command: + - "sleep" + - "infinity" diff --git a/docs/migrate-to-antrea.md b/docs/migrate-to-antrea.md new file mode 100644 index 00000000000..8c2adf5ae8f --- /dev/null +++ b/docs/migrate-to-antrea.md @@ -0,0 +1,85 @@ +# Migrate from other CNI to Antrea + +This document describes how to migrate from other CNIs to Antrea. Currently, we support migration from Calico and +Flannel. +As for Calico, NetworkPolicy conversion is conditional supported. As for Flannel, only Pod networking migration is +supported. + +With the help of migration [script](../hack/migrate-to-antrea.sh) ant `antctl` subcommand `migrate`, the migration +process is fully automated. +The migration process is divided into four steps: + +1. Install Antrea in the cluster. +2. If necessary, convert NetworkPolicy from other CNI to Antrea. +3. Restart all Pods in the cluster in-place. +4. Uninstall the old CNI. + +## Install Antrea + +The script will check the requirement of CNI migration at first. If we are migrating from Calico, the script will list +all Calico NetworkPolicy to check if there is any unsupported feature. If we are migrating from Flannel, the script will +check if Antrea is installed in the cluster. If Antrea is already installed, the script will exit with an error message. + +## Convert NetworkPolicy + +If the old CNI is Calico, the script will convert Calico NetworkPolicy to Antrea NetworkPolicy. The conversion is based +on +Calico NetworkPolicy CRD, so the script will check if the Calico APIServer is running in the cluster. If the Calico +APIServer +is not running, the script will exit with an error message. If the Calico APIServer is running, the script will convert +Calico +NetworkPolicy to Antrea NetworkPolicy using `antctl migrate convert-networkpolicy`. If the conversion fails, the script +will exit +with an error message. + +Please Note: Not all Calico NetworkPolicy features are supported by Antrea NetworkPolicy. Please refer to the following +table for +the unsupported features: + +| Calico NetworkPolicy Feature | Antrea NetworkPolicy Support | +|---------------------------------------------------------|------------------------------| +| "or" expression in any selector | NOT | +| "()" expression in any selector | NOT | +| "starts with" expression in any selector | NOT | +| "ends with" expression in any selector | NOT | +| `spec.PreDNAT` | NOT | +| `spec.ApplyOnForward` | NOT | +| `spec.DoNotTrack` | NOT | +| `spec.Ingress.ICMP` or `spec.Egress.ICMP` | NOT | +| `spec.Ingress.NotICMP` or `spec.Egress.NotICMP` | NOT | +| `spec.Ingress.NotProtocol` or `spec.Egress.NotProtocol` | NOT | +| `spec.Ingress.NotPorts` or `spec.Egress.NotPorts` | NOT | +| `spec.Ingress.Metadata` or `spec.Egress.Metadata` | NOT | +| `spec.Ingress.NotSelector` or `spec.Egress.NotSelector` | NOT | +| `spec.Ingress.HTTP` or `spec.Egress.HTTP` | NOT | +| `spec.Ingress.Nets` or `spec.Egress.Nets` | NOT | +| `spec.Ingress.NotNets` or `spec.Egress.NotNets` | NOT | +| All other features | YES | + +## Restart Pods + +After Antrea is installed in the cluster, the script will restart all Pods in the cluster in-place by deploying a +DaemonSet named `antrea-migrator`, which will run a Pod on each Node. The Pod will kill all Pods' containerd task on the +Node, and the containerd task will be restarted by the containerd service. In this way, all Pods in the cluster will be +restarted +in-place and do not need to be rescheduled and recreated. + +The restart result is as follows: + +```bash +$ kubectl get pod -n migrate-test -o wide +NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES +migrate-exmaple-6d6b97f96b-29qbq 1/1 Running 1 (24s ago) 2m5s 10.10.1.3 test-worker +migrate-exmaple-6d6b97f96b-dqx2g 1/1 Running 1 (23s ago) 2m5s 10.10.1.6 test-worker +migrate-exmaple-6d6b97f96b-jpflg 1/1 Running 1 (23s ago) 2m5s 10.10.1.5 test-worker +``` + +## Uninstall old CNI + +After all Pods are restarted, the script will uninstall the old CNI by using `kubectl delete -f `. If the +old CNI +is Calico, the script will also delete the Calico iptables rules on each Node by following commands: + +```bash +kubectl exec -n kube-system {ANTREA_AGENT} -- /bin/bash -c 'iptables-save | grep -v cali | iptables-restore' --kubeconfig $KUBECONFIG +``` \ No newline at end of file diff --git a/go.mod b/go.mod index cd2e5d4b4fd..f56b45d7304 100644 --- a/go.mod +++ b/go.mod @@ -42,6 +42,7 @@ require ( github.com/onsi/ginkgo/v2 v2.13.0 github.com/onsi/gomega v1.29.0 github.com/pkg/sftp v1.13.6 + github.com/projectcalico/api v0.0.0-20230602153125-fb7148692637 github.com/prometheus/client_golang v1.17.0 github.com/prometheus/common v0.45.0 github.com/sirupsen/logrus v1.9.3 @@ -74,7 +75,7 @@ require ( k8s.io/component-base v0.26.4 k8s.io/klog/v2 v2.100.1 k8s.io/kube-aggregator v0.26.4 - k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 + k8s.io/kube-openapi v0.0.0-20230303024457-afdc3dddf62d k8s.io/kubectl v0.26.4 k8s.io/kubelet v0.26.4 k8s.io/utils v0.0.0-20230209194617-a36077c30491 @@ -140,7 +141,7 @@ require ( github.com/google/cel-go v0.12.6 // indirect github.com/google/gnostic v0.5.7-v3refs // indirect github.com/google/go-cmp v0.6.0 // indirect - github.com/google/gofuzz v1.1.0 // indirect + github.com/google/gofuzz v1.2.0 // indirect github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect diff --git a/go.sum b/go.sum index 8c4cf2156b0..9b147cc540a 100644 --- a/go.sum +++ b/go.sum @@ -605,8 +605,9 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-containerregistry v0.5.1/go.mod h1:Ct15B4yir3PLOP5jsy0GNeYVaIZs/MK/Jz5any1wFW0= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -960,6 +961,8 @@ github.com/pkg/sftp v1.13.6/go.mod h1:tz1ryNURKu77RL+GuCzmoJYxQczL3wLNNpPWagdg4Q github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= +github.com/projectcalico/api v0.0.0-20230602153125-fb7148692637 h1:F48and+6vKJsRMl95Y/XKVik0Kwhos8YShTH9Fsdqlw= +github.com/projectcalico/api v0.0.0-20230602153125-fb7148692637/go.mod h1:d3yVTVhVHDawgeKrru/ZZD8QLEtiKQciUaAwnua47Qg= github.com/prometheus/client_golang v0.0.0-20180209125602-c332b6f63c06/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= @@ -1841,8 +1844,8 @@ k8s.io/kube-openapi v0.0.0-20200121204235-bf4fb3bd569c/go.mod h1:GRQhZsXIAJ1xR0C k8s.io/kube-openapi v0.0.0-20200410145947-61e04a5be9a6/go.mod h1:GRQhZsXIAJ1xR0C9bd8UpWHZ5plfAS9fzPjJuQ6JL3E= k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H64/qeyjQoUZhGpeFDVdxjTeEVN2o= k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= -k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 h1:+70TFaan3hfJzs+7VK2o+OGxg8HsuBr/5f6tVAjDu6E= -k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280/go.mod h1:+Axhij7bCpeqhklhUTe3xmOn6bWxolyZEeyaFpjGtl4= +k8s.io/kube-openapi v0.0.0-20230303024457-afdc3dddf62d h1:VcFq5n7wCJB2FQMCIHfC+f+jNcGgNMar1uKd6rVlifU= +k8s.io/kube-openapi v0.0.0-20230303024457-afdc3dddf62d/go.mod h1:y5VtZWM9sHHc2ZodIH/6SHzXj+TPU5USoA8lcIeKEKY= k8s.io/kubectl v0.26.4 h1:A0Oa0u/po4KxXnXsNCOwLojAe9cQR3TJNJabEIf7U1w= k8s.io/kubectl v0.26.4/go.mod h1:cWtp/+I4p+h5En3s2zO1zCry9v3/6h37EQ2tF3jNRnM= k8s.io/kubelet v0.26.4 h1:SEQPfjN4lu4uL9O8NdeN7Aum3liQ4kOnp/yC3jMRMUo= diff --git a/hack/migrate-to-antrea.sh b/hack/migrate-to-antrea.sh new file mode 100755 index 00000000000..b68e772a7b3 --- /dev/null +++ b/hack/migrate-to-antrea.sh @@ -0,0 +1,221 @@ +#!/usr/bin/env bash + +# Copyright 2023 Antrea Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script is used for migrating other CNIs to Antrea CNI. + +set -e + +_usage="Usage: $0 [--from] [--networkpolicy] [--dry-run] [--back-up] + [--output-dir] [--kubeconfig] [--calico-manifest] [--flannel-manifest] + [--antrea-manifest] [-h | --help] +Migrate other CNIs to Antrea CNI, and convert NetworkPolicy to Antrea NetworkPolicy. +During the migration, the workloads will be restarted and the network will be interrupted. + --from The CNI to be migrated from. Only support 'calico', 'flannel' now. + --networkpolicy Convert NetworkPolicy to Antrea NetworkPolicy or not. + --dry-run Print the commands that would be executed, but do not execute them. + --back-up Back up the original CNI configuration files and network policies. + --output-dir Output directory for the backup files. + --kubeconfig Path to kubeconfig file. + --calico-manifest Path to the Calico manifest file, if not specified, the latest manifest will be used. + --flannel-manifest Path to the Flannel manifest file, if not specified, the latest manifest will be used. + --antrea-manifest Path to the Antrea manifest file, if not specified, the latest manifest will be used. +" + +_calico_warning_message="WARNING: You are trying to migrate from Calico to Antrea. Please NOTE: +During migration, the workloads will restart and the network will be interrupted. +If you are using Kubernetes origin Networkpolicy, all the network policies will still work well +after migration. +If you are using Calico NetworkPolicy or Calico GlobalNetworkPolicy, please NOTE that only PART of +the network policies could be converted to Antrea NetworkPolicy and Antrea ClusterNetworkPolicy. +Please check the migration document for details." + +function print_usage { + echo "$_usage" +} + +FROM="" +NETWORK_POLICY=false +DRY_RUN=false +BACK_UP=false +OUTPUT_DIR="" +KUBECONFIG="$HOME/.kube/config" +PLATFORM=$(uname | tr '[:upper:]' '[:lower:]') +CALICO_MANIFEST="https://docs.projectcalico.org/manifests/calico.yaml" +FLANNEL_MANIFEST="https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml" +ANTREA_MANIFEST="./build/yamls/antrea.yml" + +while [[ $# -gt 0 ]]; do + key="$1" + + case $key in + --from) + FROM="$2" + shift 2 + ;; + --networkpolicy) + NETWORK_POLICY=true + shift + ;; + --dry-run) + DRY_RUN=true + shift + ;; + --back-up) + BACK_UP=true + shift + ;; + --output-dir) + OUTPUT_DIR="$2" + shift 2 + ;; + --kubeconfig) + KUBECONFIG="$2" + shift 2 + ;; + -h | --help) + print_usage + exit 0 + ;; + *) # unknown option + echo "Unknown option $1" + exit 1 + ;; + esac +done + +if [ "$FROM" == "calico" ]; then + echo "$_calico_warning_message" + # input to continue + echo "========== Input 'yes' to continue ==========" + read -r input + if [ "$input" != "yes" ]; then + echo "========== Aborting ==========" + exit 1 + fi +fi +echo "========== Building antctl ==========" +make antctl +mv ./bin/antctl-${PLATFORM} ./bin/antctl + +echo "========== Checking the environment ==========" +if [ $DRY_RUN == true ]; then + echo "./bin/antctl migrate check-env --from $FROM" +else + ./bin/antctl migrate check-env --from $FROM + if [ $? -ne 0 ]; then + echo "Failed to check the environment" + exit 1 + fi +fi + +if [ "$backup" == true ]; then + if [ "$OUTPUT_DIR" == "" ]; then + echo "Please specify the output directory for the backup files" + exit 1 + fi + echo "========== Backing up the original CNI configuration files and network policies ==========" + if [ $DRY_RUN == true ]; then + echo "kubectl get networkpolicies.crd.projectcalico.org --all-namespaces --kubeconfig $KUBECONFIG -o yaml >$OUTPUT_DIR/networkpolicies.yaml" + echo "kubectl get globalnetworkpolicies --all-namespaces --kubeconfig $KUBECONFIG -o yaml >$OUTPUT_DIR/globalnetworkpolicies.yaml" + else + kubectl get networkpolicies.crd.projectcalico.org --all-namespaces --kubeconfig "$KUBECONFIG" -o yaml >"$OUTPUT_DIR"/networkpolicies.yaml + kubectl get globalnetworkpolicies --all-namespaces --kubeconfig "$KUBECONFIG" -o yaml >"$OUTPUT_DIR"/globalnetworkpolicies.yaml + fi +fi + +echo "========== Deploying Antrea ========== +========== Warning: The Service might be unavailable after deploying Antrea to your cluster ========== +========== Please make sure that you want to continue ========== +========== Input 'yes' to continue ==========" +read -r input +if [ "$input" != "yes" ]; then + echo "========== Aborting ==========" + exit 1 +fi + +if [ $DRY_RUN == true ]; then + echo "kubectl apply -f $ANTREA_MANIFEST --kubeconfig $KUBECONFIG" +else + trap "kubectl delete -f $ANTREA_MANIFEST" ERR + kubectl apply -f $ANTREA_MANIFEST --kubeconfig "$KUBECONFIG" +fi +echo "========== Waiting for Antrea to be ready ==========" +if [ $DRY_RUN == true ]; then + echo "kubectl rollout status ds/antrea-agent -n kube-system --timeout=300s --kubeconfig $KUBECONFIG" +else + kubectl rollout status ds/antrea-agent -n kube-system --timeout=300s --kubeconfig "$KUBECONFIG" +fi +echo "========== Antrea is ready ==========" +if [ $NETWORK_POLICY == true ]; then + echo "========== Converting NetworkPolicy to Antrea NetworkPolicy ==========" + if [ $DRY_RUN == true ]; then + echo "./bin/antctl migrate convert-networkpolicy --from $FROM --kubeconfig $KUBECONFIG" + else + ./bin/antctl migrate convert-networkpolicy --from "$FROM" --kubeconfig "$KUBECONFIG" + fi +fi +echo "========== Restarting the Sandboxes ==========" +if [ $DRY_RUN == true ]; then + echo "kubectl apply -f ./build/yamls/antrea-migrator.yml --kubeconfig $KUBECONFIG" +else + trap "kubectl delete -f ./build/yamls/antrea-migrator.yml" EXIT + kubectl apply -f ./build/yamls/antrea-migrator.yml --kubeconfig "$KUBECONFIG" +fi +echo "========== Waiting for sandboxes restart to be finished ==========" +if [ $DRY_RUN == true ]; then + echo "kubectl rollout status ds/antrea-migrator -n kube-system --timeout=300s --kubeconfig $KUBECONFIG" +else + kubectl rollout status ds/antrea-migrator -n kube-system --timeout=300s --kubeconfig "$KUBECONFIG" +fi + +echo "========== The restart process is finished, begin to clean former CNI ==========" + +if [ $FROM == "calico" ]; then + echo "========== Cleaning Calico ==========" + if [ $DRY_RUN == true ]; then + echo "kubectl delete -f $CALICO_MANIFEST --kubeconfig $KUBECONFIG" + else + kubectl delete -f $CALICO_MANIFEST --kubeconfig "$KUBECONFIG" + fi +elif [ $FROM == "flannel" ]; then + echo "========== Cleaning Flannel ==========" + if [ $DRY_RUN == true ]; then + echo "kubectl delete -f $FLANNEL_MANIFEST" + else + kubectl delete -f $FLANNEL_MANIFEST + fi +else + echo "Unsupported CNI: $FROM" + exit 1 +fi + +echo "========== Cleaning iptabes rules ==========" +antrea_agents=$(kubectl get pods -n kube-system -l component=antrea-agent -o jsonpath='{.items[*].metadata.name}' --kubeconfig "$KUBECONFIG") +if [ $DRY_RUN == true ]; then + echo "kubectl exec -n kube-system {ANTREA_AGENT} -- /bin/bash -c 'iptables-save | grep -v cali | iptables-restore' --kubeconfig $KUBECONFIG" +else + if [ $FROM == "calico" ]; then + for agent in $antrea_agents; do + kubectl exec -n kube-system "$agent" -- /bin/bash -c 'iptables-save | grep -v cali | iptables-restore' --kubeconfig "$KUBECONFIG" + done + else + for agent in $antrea_agents; do + kubectl exec -n kube-system "$agent" -- /bin/bash -c 'ip link delete cni0' --kubeconfig "$KUBECONFIG" + kubectl exec -n kube-system "$agent" -- /bin/bash -c 'ip link delete flannel.1' --kubeconfig "$KUBECONFIG" + kubectl exec -n kube-system "$agent" -- /bin/bash -c 'iptables-save | grep -v flannel | iptables-restore' --kubeconfig "$KUBECONFIG" + done + fi +fi diff --git a/pkg/antctl/antctl.go b/pkg/antctl/antctl.go index 2988c9b67f2..69e25a047a1 100644 --- a/pkg/antctl/antctl.go +++ b/pkg/antctl/antctl.go @@ -27,6 +27,8 @@ import ( "antrea.io/antrea/pkg/agent/openflow" fallbackversion "antrea.io/antrea/pkg/antctl/fallback/version" "antrea.io/antrea/pkg/antctl/raw/featuregates" + "antrea.io/antrea/pkg/antctl/raw/migrate/checkenv" + "antrea.io/antrea/pkg/antctl/raw/migrate/convertnetworkpolicy" "antrea.io/antrea/pkg/antctl/raw/multicluster" "antrea.io/antrea/pkg/antctl/raw/proxy" "antrea.io/antrea/pkg/antctl/raw/set" @@ -686,6 +688,18 @@ $ antctl get podmulticaststats pod -n namespace`, supportController: false, commandGroup: upgrade, }, + { + cobraCommand: checkenv.Command, + supportAgent: false, + supportController: false, + commandGroup: migrate, + }, + { + cobraCommand: convertnetworkpolicy.Command, + supportAgent: false, + supportController: false, + commandGroup: migrate, + }, }, codec: scheme.Codecs, } diff --git a/pkg/antctl/command_definition.go b/pkg/antctl/command_definition.go index 1f6cc36665a..b1c77695956 100644 --- a/pkg/antctl/command_definition.go +++ b/pkg/antctl/command_definition.go @@ -73,6 +73,7 @@ const ( query mc upgrade + migrate ) var groupCommands = map[commandGroup]*cobra.Command{ @@ -96,6 +97,11 @@ var groupCommands = map[commandGroup]*cobra.Command{ Short: "Sub-commands for upgrade operations", Long: "Sub-commands for upgrade operations", }, + migrate: { + Use: "migrate", + Short: "Sub-commands for CNI migration operations", + Long: "Sub-commands for CNI migration operations", + }, } type endpointResponder interface { diff --git a/pkg/antctl/command_list.go b/pkg/antctl/command_list.go index e60f381e435..4ed6141c4bb 100644 --- a/pkg/antctl/command_list.go +++ b/pkg/antctl/command_list.go @@ -64,7 +64,8 @@ func (cl *commandList) applyToRootCommand(root *cobra.Command, client AntctlClie (runtime.Mode == runtime.ModeController && cmd.supportController) || (runtime.Mode == runtime.ModeFlowAggregator && cmd.supportFlowAggregator) || (!runtime.InPod && cmd.commandGroup == mc) || - (!runtime.InPod && cmd.commandGroup == upgrade) { + (!runtime.InPod && cmd.commandGroup == upgrade) || + (!runtime.InPod && cmd.commandGroup == migrate) { if groupCommand, ok := groupCommands[cmd.commandGroup]; ok { groupCommand.AddCommand(cmd.cobraCommand) } else { diff --git a/pkg/antctl/raw/migrate/checkenv/command.go b/pkg/antctl/raw/migrate/checkenv/command.go new file mode 100644 index 00000000000..e02c7459298 --- /dev/null +++ b/pkg/antctl/raw/migrate/checkenv/command.go @@ -0,0 +1,290 @@ +// Copyright 2023 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package checkenv + +import ( + "context" + "fmt" + "strings" + + v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + calicoclient "github.com/projectcalico/api/pkg/client/clientset_generated/clientset" + "github.com/spf13/cobra" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/klog/v2" + + "antrea.io/antrea/pkg/antctl/raw" + "antrea.io/antrea/pkg/antctl/raw/migrate" +) + +var ( + Command *cobra.Command + options *checkEnvOptions +) + +type checkEnvOptions struct { + from string + networkPolicy bool +} + +func (o *checkEnvOptions) validateAndComplete() error { + if o.from == "" { + return fmt.Errorf("CNI is not specified") + } + for _, cni := range migrate.SupportedCNIs { + if o.from == cni { + klog.Infof("Migrating from %s", o.from) + return nil + } + } + + return fmt.Errorf("unsupported CNI: %s", o.from) +} + +type envChecker struct { + k8sClient *kubernetes.Clientset + calicoClient *calicoclient.Clientset +} + +func newChecker(kubeconfig *rest.Config) (*envChecker, error) { + k8sClient, _, err := raw.SetupClients(kubeconfig) + if err != nil { + return nil, err + } + calicoClient, err := calicoclient.NewForConfig(kubeconfig) + if err != nil { + return nil, err + } + return &envChecker{ + k8sClient: k8sClient, + calicoClient: calicoClient, + }, nil +} + +var example = strings.Trim(` + Check the environment for migrating Calico to Antrea without NetworkPolicy conversion + $ antctl migrate check-env --from calico + + Migrate from Calico to Antrea and convert the NetworkPolicies + $ antctl migrate check-env --from calico --networkpolicy +`, "\n") + +func init() { + Command = &cobra.Command{ + Use: "check-env", + Short: "Check the environment for migrating to Antrea", + Long: "Check the environment for migrating to Antrea, including the CNI and NetworkPolicies.", + Example: example, + RunE: runE, + Args: cobra.NoArgs, + } + + o := &checkEnvOptions{} + options = o + Command.Flags().StringVar(&o.from, "from", "", "The CNI to migrate from. Supported values: calico, flannel.") + Command.Flags().BoolVar(&o.networkPolicy, "network-policy", false, "If true, check NetworkPolicies.") +} + +func runE(cmd *cobra.Command, _ []string) error { + if err := options.validateAndComplete(); err != nil { + return err + } + kubeconfig, err := raw.ResolveKubeconfig(cmd) + if err != nil { + return err + } + c, err := newChecker(kubeconfig) + if err != nil { + return err + } + if err := c.checkEnv(options.from); err != nil { + return err + } + if options.networkPolicy { + if err := c.checkNetworkPolicy(); err != nil { + return err + } + } + klog.Infof("Migration from %s to Antrea is supported", options.from) + return nil +} + +func (c *envChecker) checkEnv(cni string) error { + // Check if Antrea is installed + daemonSets, err := c.k8sClient.AppsV1().DaemonSets(metav1.NamespaceSystem).List(context.TODO(), metav1.ListOptions{ + LabelSelector: "app=antrea", + }) + if err != nil { + klog.Errorf("Failed to list Antrea DaemonSets: %v", err) + return err + } + if len(daemonSets.Items) > 0 { + return fmt.Errorf("Antrea is already installed") + } + + // Check if Calico is installed + if cni == migrate.CalicoCNI { + calicoAPIServer, err := c.k8sClient.AppsV1().Deployments("").List(context.TODO(), metav1.ListOptions{ + LabelSelector: "k8s-app=calico-apiserver", + }) + if err != nil { + klog.Errorf("Failed to list Deployments: %v", err) + return err + } + if len(calicoAPIServer.Items) == 0 { + return fmt.Errorf("calico-apiserver is not installed") + } + if err := c.checkNetworkPolicy(); err != nil { + return err + } + } + + return nil +} + +func (c *envChecker) checkNetworkPolicy() error { + globalNetworkPolicies, err := c.calicoClient.ProjectcalicoV3().GlobalNetworkPolicies().List(context.TODO(), metav1.ListOptions{}) + if err != nil { + klog.Errorf("Failed to list GlobalNetworkPolicies: %v", err) + return err + } + for _, gnp := range globalNetworkPolicies.Items { + klog.Infof("Checking GlobalNetworkPolicy %s", gnp.Name) + // Calico GlobalNetworkPolicy/NetworkPolicy supports some complex selectors, which might not be supported by Antrea + // directly. For example, Calico supports the following selectors: + // ... + // selector: role == 'db' && (os == 'linux' || os == 'windows') + // ... + // Antrea DOES NOT support "()" expression in selectors. So the above selectors are not supported by Antrea. + // The following selectors are supported by Antrea: + // ... + // selector: role == 'db' && os == 'linux' + // or + // selector: app == 'nginx && has(os)' + // + // More details can be found in the following link: + // https://docs.tigera.io/calico/latest/reference/resources/networkpolicy#selectors + if err := checkSelector(gnp.Spec.Selector); err != nil { + return err + } + if err := checkSelector(gnp.Spec.NamespaceSelector); err != nil { + return err + } + if err := checkSelector(gnp.Spec.ServiceAccountSelector); err != nil { + return err + } + if gnp.Spec.PreDNAT { + return fmt.Errorf("GlobalNetworkPolicy %s has set field spec.preDNAT, which is not supported by Antrea", gnp.Name) + } + if gnp.Spec.DoNotTrack { + return fmt.Errorf("GlobalNetworkPolicy %s has set field spec.doNotTrack, which is not supported by Antrea", gnp.Name) + } + if gnp.Spec.ApplyOnForward { + return fmt.Errorf("GlobalNetworkPolicy %s has set field spec.applyOnForward, which is not supported by Antrea", gnp.Name) + } + if err := checkCalicoRules(gnp.Spec.Egress, gnp.Name); err != nil { + return err + } + if err := checkCalicoRules(gnp.Spec.Ingress, gnp.Name); err != nil { + return err + } + } + + networkPolicies, err := c.calicoClient.ProjectcalicoV3().NetworkPolicies("").List(context.TODO(), metav1.ListOptions{}) + if err != nil { + klog.Errorf("Failed to list NetworkPolicies: %v", err) + return err + } + for _, np := range networkPolicies.Items { + if np.Spec.ServiceAccountSelector != "" { + return fmt.Errorf("NetworkPolicy %s has set field spec.serviceAccountSelector, which is not supported by Antrea", np.Name) + } + if err := checkCalicoRules(np.Spec.Egress, np.Name); err != nil { + return err + } + if err := checkCalicoRules(np.Spec.Ingress, np.Name); err != nil { + return err + } + } + + return nil +} + +func checkCalicoRules(rules []v3.Rule, gnpName string) error { + for _, rule := range rules { + if rule.ICMP != nil { + return fmt.Errorf("GlobalNetworkPolicy/NetworkPolicy %s has set field spec.egress.icmp or spec.ingress.icmp,"+ + "which is not supported by Antrea", gnpName) + } + if rule.NotICMP != nil { + return fmt.Errorf("GlobalNetworkPolicy/NetworkPolicy %s has set field spec.egress.notICMP or spec.ingress.icmp,"+ + "which is not supported by Antrea", gnpName) + } + if rule.NotProtocol != nil { + return fmt.Errorf("GlobalNetworkPolicy/NetworkPolicy %s has set field spec.egress.notProtocol or spec.ingress.notProtocol,"+ + "which is not supported by Antrea", gnpName) + } + if rule.Metadata != nil { + return fmt.Errorf("GlobalNetworkPolicy/NetworkPolicy %s has set field spec.egress.metadata or spec.ingress.metadata,"+ + "which is not supported by Antrea", gnpName) + } + if rule.HTTP != nil { + return fmt.Errorf("GlobalNetworkPolicy/NetworkPolicy %s has set field spec.egress.http or spec.ingress.http,"+ + "which is not supported by Antrea", gnpName) + } + if rule.Source.NotSelector != "" || rule.Destination.NotSelector != "" { + return fmt.Errorf("GlobalNetworkPolicy/NetworkPolicy %s has set field spec.egress.notSelector or spec.ingress.notSelector,"+ + "which is not supported by Antrea", gnpName) + } + + // Maybe we can support Nets by converting them to Groups + if rule.Source.NotNets != nil || rule.Destination.NotNets != nil { + return fmt.Errorf("GlobalNetworkPolicy/NetworkPolicy %s has set field spec.egress.notNets or spec.ingress.notNets,"+ + "which is not supported by Antrea", gnpName) + } + if rule.Source.Nets != nil || rule.Destination.Nets != nil { + return fmt.Errorf("GlobalNetworkPolicy/NetworkPolicy %s has set field spec.egress.nets or spec.ingress.nets,"+ + "which is not supported by Antrea", gnpName) + } + if rule.Source.NotPorts != nil || rule.Destination.NotPorts != nil { + return fmt.Errorf("GlobalNetworkPolicy/NetworkPolicy %s has set field spec.egress.notPorts or spec.ingress.notPorts,"+ + "which is not supported by Antrea", gnpName) + } + if err := checkSelector(rule.Source.Selector); err != nil { + return err + } + if err := checkSelector(rule.Source.NamespaceSelector); err != nil { + return err + } + } + + return nil +} + +// checkSelector checks if the selector is supported by Antrea. +func checkSelector(selector string) error { + if strings.Contains(selector, "||") { + return fmt.Errorf("selector %s is not supported by Antrea", selector) + } + selector = strings.ReplaceAll(selector, " ", "") + brackets := strings.Count(selector, "(") + has := strings.Count(selector, "has(") + if brackets != has { + return fmt.Errorf("selector %s is not supported by Antrea", selector) + } + return nil +} diff --git a/pkg/antctl/raw/migrate/checkenv/command_test.go b/pkg/antctl/raw/migrate/checkenv/command_test.go new file mode 100644 index 00000000000..9ef51c41c47 --- /dev/null +++ b/pkg/antctl/raw/migrate/checkenv/command_test.go @@ -0,0 +1,61 @@ +package checkenv + +import ( + "testing" +) + +func TestCheckSelector(t *testing.T) { + testCases := []struct { + name string + selector string + wantErr bool + }{ + { + name: "use expression '||'", + selector: "a == 'a' || a == 'b'", + wantErr: true, + }, + { + name: "use expression '&&'", + selector: "a == 'a' && a == 'b'", + wantErr: false, + }, + { + name: "use expression 'in'", + selector: "a in {'a', 'b'}", + wantErr: false, + }, + { + name: "use expression 'not in'", + selector: "a not in {'a', 'b'}", + wantErr: false, + }, + { + name: "use expression '=='", + selector: "a == 'a'", + wantErr: false, + }, + { + name: "use expression '!='", + selector: "a != 'a'", + wantErr: false, + }, + { + name: "use expression 'has'", + selector: "a has ('a')", + wantErr: false, + }, + { + name: "use expression '()'", + selector: "a == 'b' && ( c == 'd' && e == 'f')", + wantErr: true, + }, + } + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + if err := checkSelector(tt.selector); (err != nil) != tt.wantErr { + t.Errorf("checkSelector() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} diff --git a/pkg/antctl/raw/migrate/common.go b/pkg/antctl/raw/migrate/common.go new file mode 100644 index 00000000000..ad5255524f6 --- /dev/null +++ b/pkg/antctl/raw/migrate/common.go @@ -0,0 +1,28 @@ +// Copyright 2023 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package migrate + +const ( + CalicoCNI = "calico" + FlannelCNI = "flannel" +) + +var ( + SupportedCNIs = []string{CalicoCNI, FlannelCNI} +) + +func ToPtr[T any](t T) *T { + return &t +} diff --git a/pkg/antctl/raw/migrate/convertnetworkpolicy/command.go b/pkg/antctl/raw/migrate/convertnetworkpolicy/command.go new file mode 100644 index 00000000000..4e2e8f2b76f --- /dev/null +++ b/pkg/antctl/raw/migrate/convertnetworkpolicy/command.go @@ -0,0 +1,414 @@ +// Copyright 2023 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License.package networkpolicy + +package convertnetworkpolicy + +import ( + "context" + "fmt" + "strings" + + v3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + calicoclient "github.com/projectcalico/api/pkg/client/clientset_generated/clientset" + "github.com/spf13/cobra" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/client-go/rest" + "k8s.io/klog/v2" + + "antrea.io/antrea/pkg/antctl/raw" + "antrea.io/antrea/pkg/antctl/raw/migrate" + "antrea.io/antrea/pkg/apis/crd/v1beta1" + antrea "antrea.io/antrea/pkg/client/clientset/versioned" +) + +const ( + convertFromCalicoAnnotation = "convert-from-calico" + + defaultPriority = 1 +) + +type convertOptions struct { + from string +} + +type converter struct { + antreaClient *antrea.Clientset + calicoClient *calicoclient.Clientset +} + +var ( + Command *cobra.Command + options *convertOptions +) + +func (o *convertOptions) validateAndComplete() error { + if o.from != "calico" { + return fmt.Errorf("unsupported CNI: %s", o.from) + } + return nil +} + +func newConverter(kubeconfig *rest.Config) (*converter, error) { + _, antreaClient, err := raw.SetupClients(kubeconfig) + if err != nil { + return nil, err + } + calicoClient, err := calicoclient.NewForConfig(kubeconfig) + if err != nil { + return nil, err + } + return &converter{ + antreaClient: antreaClient, + calicoClient: calicoClient, + }, nil +} + +var migrateCommandExample = strings.Trim(` + Convert Calico NetworkPolicies to Antrea NetworkPolicies: + $ antctl migrate convert-networkpolicy --from calico +`, "\n") + +func init() { + Command = &cobra.Command{ + Use: "convert-networkpolicy", + Short: "Convert NetworkPolicy from other CNI to Antrea", + Long: "Convert NetworkPolicy from other CNI to Antrea", + Example: migrateCommandExample, + RunE: runE, + Args: cobra.NoArgs, + } + + o := &convertOptions{} + options = o + Command.Flags().StringVar(&o.from, "from", "", "The CNI to migrate from. Supported values: calico.") +} + +func runE(cmd *cobra.Command, _ []string) error { + if err := options.validateAndComplete(); err != nil { + return err + } + kubeconfig, err := raw.ResolveKubeconfig(cmd) + if err != nil { + return err + } + c, err := newConverter(kubeconfig) + if err != nil { + return err + } + if err := c.convertNetworkPolicy(); err != nil { + return err + } + + return nil +} + +func (c *converter) convertNetworkPolicy() error { + if options.from == "calico" { + return c.convertCalicoNetworkPolicy() + } + return nil +} + +func (c *converter) convertCalicoNetworkPolicy() error { + if err := c.convertGlobalNetworkPolicy(); err != nil { + klog.Errorf("Failed to convert Calico GlobalNetworkPolicy: %v", err) + return err + } + if err := c.convertCalicoNamespacedNetworkPolicy(); err != nil { + klog.Errorf("Failed to convert Calico NamespacedNetworkPolicy: %v", err) + return err + } + klog.Infof("Successfully converted Calico NetworkPolicy to Antrea NetworkPolicy") + return nil +} + +func (c *converter) convertCalicoNamespacedNetworkPolicy() error { + nps, err := c.calicoClient.ProjectcalicoV3().NetworkPolicies("").List(context.Background(), metav1.ListOptions{}) + if err != nil { + klog.Errorf("Failed to list Calico NetworkPolicies: %v", err) + return err + } + for _, np := range nps.Items { + klog.Infof("Converting NetworkPolicy %s", np.Name) + anp := v1beta1.NetworkPolicy{ + ObjectMeta: metav1.ObjectMeta{ + Name: np.Name, + Namespace: np.Namespace, + Annotations: map[string]string{ + convertFromCalicoAnnotation: "true", + }, + }, + } + + if anp.Spec.AppliedTo, err = convertSelectorToAntreaAppliedTo(np.Spec.Selector, ""); err != nil { + return err + } + if anp.Spec.Egress, err = convertCalicoRuleToAntreaRule(np.Spec.Egress); err != nil { + return err + } + if anp.Spec.Ingress, err = convertCalicoRuleToAntreaRule(np.Spec.Ingress); err != nil { + return err + } + } + return nil +} + +func (c *converter) convertGlobalNetworkPolicy() error { + gnps, err := c.calicoClient.ProjectcalicoV3().GlobalNetworkPolicies().List(context.Background(), metav1.ListOptions{}) + if err != nil { + klog.Errorf("Failed to list GlobalNetworkPolicies: %v", err) + return err + } + for _, gnp := range gnps.Items { + klog.Infof("Converting GlobalNetworkPolicy %s", gnp.Name) + acnp := v1beta1.ClusterNetworkPolicy{ + ObjectMeta: metav1.ObjectMeta{ + Name: gnp.Name, + Annotations: map[string]string{ + convertFromCalicoAnnotation: "true", + }, + }, + } + + if acnp.Spec.AppliedTo, err = convertSelectorToAntreaAppliedTo(gnp.Spec.Selector, gnp.Spec.NamespaceSelector); err != nil { + return err + } + if acnp.Spec.Ingress, err = convertCalicoRuleToAntreaRule(gnp.Spec.Ingress); err != nil { + return err + } + if acnp.Spec.Egress, err = convertCalicoRuleToAntreaRule(gnp.Spec.Egress); err != nil { + return err + } + if gnp.Spec.Order == nil { + acnp.Spec.Priority = defaultPriority + } else { + acnp.Spec.Priority = *gnp.Spec.Order + } + + if _, err := c.antreaClient.CrdV1beta1().ClusterNetworkPolicies().Create(context.Background(), &acnp, metav1.CreateOptions{}); err != nil { + klog.Errorf("Failed to create ClusterNetworkPolicy %s: %v", acnp.Name, err) + return err + } + } + return nil +} + +func convertCalicoRuleToAntreaRule(rule []v3.Rule) ([]v1beta1.Rule, error) { + var antreaRules []v1beta1.Rule + + for _, r := range rule { + antreaRule := v1beta1.Rule{} + switch r.Action { + case v3.Allow: + antreaRule.Action = migrate.ToPtr(v1beta1.RuleActionAllow) + case v3.Log: + antreaRule.EnableLogging = true + antreaRule.Action = migrate.ToPtr(v1beta1.RuleActionAllow) + case v3.Deny: + antreaRule.Action = migrate.ToPtr(v1beta1.RuleActionReject) + case v3.Pass: + antreaRule.Action = migrate.ToPtr(v1beta1.RuleActionPass) + } + var err error + if antreaRule.From, err = convertEntityRuleToAntreaNetworkPolicyPeer(r.Source); err != nil { + return nil, err + } + if antreaRule.To, err = convertEntityRuleToAntreaNetworkPolicyPeer(r.Destination); err != nil { + return nil, err + } + if antreaRule.Ports, err = convertCalicoPortToAntreaPort(r); err != nil { + return nil, err + } + antreaRule.ToServices = convertCalicoServiceToAntreaService(r) + antreaRules = append(antreaRules, antreaRule) + } + + if len(rule) != 0 { + antreaRules = append(antreaRules, + v1beta1.Rule{Action: migrate.ToPtr(v1beta1.RuleActionReject)}) + } + return antreaRules, nil +} + +func convertCalicoPortToAntreaPort(rule v3.Rule) ([]v1beta1.NetworkPolicyPort, error) { + antreaPorts := []v1beta1.NetworkPolicyPort{} + for _, srcPort := range rule.Source.Ports { + antreaPort := v1beta1.NetworkPolicyPort{ + SourcePort: migrate.ToPtr(int32(srcPort.MinPort)), + SourceEndPort: migrate.ToPtr(int32(srcPort.MaxPort)), + } + if rule.Protocol != nil { + antreaPort.Protocol = (*v1.Protocol)(&rule.Protocol.StrVal) + } + antreaPorts = append(antreaPorts, antreaPort) + } + + for _, destPort := range rule.Destination.Ports { + antreaPort := v1beta1.NetworkPolicyPort{} + if destPort.PortName != "" { + antreaPort.Port = &intstr.IntOrString{ + Type: intstr.String, + StrVal: destPort.PortName, + } + } else { + antreaPort.Port = &intstr.IntOrString{ + Type: intstr.Int, + IntVal: int32(destPort.MinPort), + } + antreaPort.EndPort = migrate.ToPtr(int32(destPort.MaxPort)) + } + if rule.Protocol != nil { + antreaPort.Protocol = (*v1.Protocol)(&rule.Protocol.StrVal) + } + antreaPorts = append(antreaPorts, antreaPort) + } + + return antreaPorts, nil +} + +func convertCalicoServiceToAntreaService(rule v3.Rule) []v1beta1.PeerService { + if rule.Destination.Services != nil { + return []v1beta1.PeerService{ + { + Name: rule.Destination.Services.Name, + Namespace: rule.Destination.Services.Namespace, + }, + } + } + return nil +} + +func convertEntityRuleToAntreaNetworkPolicyPeer(entityRule v3.EntityRule) ([]v1beta1.NetworkPolicyPeer, error) { + var antreaNetworkPolicyPeers []v1beta1.NetworkPolicyPeer + antreaNetworkPolicyPeer := v1beta1.NetworkPolicyPeer{} + var err error + if antreaNetworkPolicyPeer.PodSelector, err = convertCalicoSelectorToLabelSelector(entityRule.Selector); err != nil { + return nil, err + } + if antreaNetworkPolicyPeer.NamespaceSelector, err = convertCalicoSelectorToLabelSelector(entityRule.NamespaceSelector); err != nil { + return nil, err + } + antreaNetworkPolicyPeers = append(antreaNetworkPolicyPeers, antreaNetworkPolicyPeer) + for _, cidr := range entityRule.Nets { + antreaNetworkPolicyPeers = append(antreaNetworkPolicyPeers, v1beta1.NetworkPolicyPeer{ + IPBlock: &v1beta1.IPBlock{ + CIDR: cidr, + }, + }) + } + return antreaNetworkPolicyPeers, nil +} + +func convertSelectorToAntreaAppliedTo(selector string, namespaceSelector string) ([]v1beta1.AppliedTo, error) { + appliedTo := &v1beta1.AppliedTo{} + var err error + if appliedTo.PodSelector, err = convertCalicoSelectorToLabelSelector(selector); err != nil { + return nil, err + } + if appliedTo.NamespaceSelector, err = convertCalicoSelectorToLabelSelector(namespaceSelector); err != nil { + return nil, err + } + + return []v1beta1.AppliedTo{*appliedTo}, nil +} + +// convertCalicoSelectorToLabelSelector parses the Calico selector string to a LabelSelector in following format: +// 1. "k == v" => LabelSelector {MatchLabels: map[string]string{"k": "v"}} +// 2. "k == v && k2 == v2" => LabelSelector {MatchLabels: map[string]string{"k": "v", "k2": "v2"}} +// 3. has(abc) => LabelSelector {MatchExpressions: []LabelSelectorRequirement{{Key: "abc", Operator: Exists}}} +// 4. !has(abc) => LabelSelector {MatchExpressions: []LabelSelectorRequirement{{Key: "abc", Operator: DoesNotExist}}} +// 5. "k in {v1, v2}" => LabelSelector {MatchExpressions: []LabelSelectorRequirement{{Key: "k", Operator: In, Values: []string{"v1", "v2"}}}} +// 6. "k not in {v1, v2}" => LabelSelector {MatchExpressions: []LabelSelectorRequirement{{Key: "k", Operator: NotIn, Values: []string{"v1", "v2"}}}} +// 7. "a != b" => LabelSelector {MatchExpressions: []LabelSelectorRequirement{{Key: "a", Operator: NotIn, Values: []string{"b"}}}} +// 8. "a == b && c != d" => LabelSelector {MatchLabels: map[string]string{"a": "b"}, MatchExpressions: []LabelSelectorRequirement{{Key: "c", Operator: NotIn, Values: []string{"d"}}}} +func convertCalicoSelectorToLabelSelector(selector string) (*metav1.LabelSelector, error) { + selector = strings.ReplaceAll( + strings.ReplaceAll(selector, " ", ""), "'", "") + conditions := strings.Split(selector, "&&") + if len(conditions) == 0 { + return nil, fmt.Errorf("invalid selector: %s", selector) + } + labelSelector := &metav1.LabelSelector{ + MatchLabels: make(map[string]string), + MatchExpressions: []metav1.LabelSelectorRequirement{}, + } + for _, condition := range conditions { + // Expression format: "k == v" + if strings.Contains(condition, "==") { + kv := strings.Split(condition, "==") + labelSelector.MatchLabels[kv[0]] = kv[1] + } + // Expression format: "k != v" + if strings.Contains(condition, "!=") { + kv := strings.Split(condition, "!=") + labelSelector.MatchExpressions = append(labelSelector.MatchExpressions, metav1.LabelSelectorRequirement{ + Key: kv[0], + Operator: metav1.LabelSelectorOpNotIn, + Values: []string{kv[1]}, + }) + } + // Expression format: "k in {v1, v2}" + if strings.Contains(condition, " in") && !strings.Contains(condition, " not") { + key, values := getKeyAndValues(condition) + labelSelector.MatchExpressions = append(labelSelector.MatchExpressions, metav1.LabelSelectorRequirement{ + Key: key, + Operator: metav1.LabelSelectorOpIn, + Values: values, + }) + } + // Expression format: "k not in {v1, v2}" + if strings.Contains(condition, " not in") { + key, values := getKeyAndValues(condition) + labelSelector.MatchExpressions = append(labelSelector.MatchExpressions, metav1.LabelSelectorRequirement{ + Key: key, + Operator: metav1.LabelSelectorOpNotIn, + Values: values, + }) + } + // Expression format: "has(k)" + if strings.Contains(condition, "has(") { + key := condition[strings.Index(condition, "(")+1 : strings.Index(condition, ")")] + labelSelector.MatchExpressions = append(labelSelector.MatchExpressions, metav1.LabelSelectorRequirement{ + Key: key, + Operator: metav1.LabelSelectorOpExists, + }) + } + // Expression format: "!has(k)" + if strings.Contains(condition, "!has(") { + key := condition[strings.Index(condition, "(")+1 : strings.Index(condition, ")")] + labelSelector.MatchExpressions = append(labelSelector.MatchExpressions, metav1.LabelSelectorRequirement{ + Key: key, + Operator: metav1.LabelSelectorOpDoesNotExist, + }) + } + } + + return labelSelector, nil +} + +// getKeyAndValues parses the condition string to key and values. +// For example, "k in {v1, v2}" will be parsed to "k" and []string{"v1", "v2"} +// and "k not in {v1, v2}" will be parsed to "k" and []string{"v1", "v2"} +func getKeyAndValues(condition string) (string, []string) { + key := condition[:strings.Index(condition, " ")] + + valuesStr := condition[strings.Index(condition, "{")+1 : strings.Index(condition, "}")] + valuesStr = strings.ReplaceAll(valuesStr, " ", "") + valuesStr = strings.ReplaceAll(valuesStr, "'", "") + values := strings.Split(valuesStr, ",") + + return key, values +} diff --git a/pkg/client/clientset/versioned/typed/crd/v1alpha1/clusternetworkpolicy.go b/pkg/client/clientset/versioned/typed/crd/v1alpha1/clusternetworkpolicy.go index 04a50350d35..ff0d314f81e 100644 --- a/pkg/client/clientset/versioned/typed/crd/v1alpha1/clusternetworkpolicy.go +++ b/pkg/client/clientset/versioned/typed/crd/v1alpha1/clusternetworkpolicy.go @@ -20,12 +20,13 @@ import ( "context" "time" - v1alpha1 "antrea.io/antrea/pkg/apis/crd/v1alpha1" - scheme "antrea.io/antrea/pkg/client/clientset/versioned/scheme" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" rest "k8s.io/client-go/rest" + + v1alpha1 "antrea.io/antrea/pkg/apis/crd/v1alpha1" + scheme "antrea.io/antrea/pkg/client/clientset/versioned/scheme" ) // ClusterNetworkPoliciesGetter has a method to return a ClusterNetworkPolicyInterface.