Support Egress using IPs from a separate subnet

By default, it's assumed that the IPs allocated from the pool are in the same subnet as the Node IPs. In some cases, users want to use IPs in different subnets as Egress IPs. Additionally, users may want to use VLAN taggaing to segment the Egress traffic and the Node traffic. The commit implements the requirements by introducing an optional field, `subnetInfo`, to the ExternalIPPool resource. The `subnetInfo` field contains the subnet attributes of the IPs in this pool. When using a different subnet: * `gateway` and `prefixLength` must be set. Antrea will route Egress traffic to the specified gateway when the destination is not in the same subnet of the Egress IP, otherwise route it to the destination directly. * Optionally, you can specify `vlan` if the underlying network is expecting it. Once set, Antrea will tag Egress traffic leaving the Egress Node with the specified VLAN ID. Correspondingly, it's expected that reply traffic towards these Egress IPs are also tagged with the specified VLAN ID when arriving the Egress Node. The implementation involves VLAN sub-interfaces and policy routing. * For a given subnet with a VLAN ID, a separate VLAN sub-interface will be created to hold the Egress IPs allocated from it. Egress traffic and its reply traffic will be sent over and received from the VLAN sub-interface for proper tagging and untagging. * For a given subnet, a separate route table will be created, routing the selected Egress traffic to the specified gateway, or to its neighbor. * For multiple Egress IPs associated allocated from the same subnet, a separate IP rule will be created for each Egress IP, matching its pkt mark and looking up the shared route table. The feature is gated by the alpha "EgressSeparateSubnet" feature gate. Signed-off-by: Quan Tian <[email protected]>
antrea-io · Jan 3, 2024 · 6ac68e8 · 6ac68e8
1 parent 327cb03
commit 6ac68e8
Show file tree

Hide file tree

Showing 47 changed files with 2,303 additions and 263 deletions.
diff --git a/.github/workflows/kind.yml b/.github/workflows/kind.yml
@@ -237,6 +237,7 @@ jobs:
             --feature-gates AllAlpha=true,AllBeta=true \
             --proxy-all \
             --node-ipam \
+            --extra-vlan \
             --multicast
       - name: Tar coverage files
         run: tar -czf test-e2e-encap-all-features-enabled-coverage.tar.gz test-e2e-encap-all-features-enabled-coverage

diff --git a/build/charts/antrea/conf/antrea-agent.conf b/build/charts/antrea/conf/antrea-agent.conf
@@ -79,6 +79,9 @@ featureGates:
 # Enable Egress traffic shaping.
 {{- include "featureGate" (dict "featureGates" .Values.featureGates "name" "EgressTrafficShaping" "default" false) }}
 
+# Allow users to allocate Egress IPs from a separate subnet different from the default Node subnet.
+{{- include "featureGate" (dict "featureGates" .Values.featureGates "name" "EgressSeparateSubnet" "default" false) }}
+
 # Name of the OpenVSwitch bridge antrea-agent will create and use.
 # Make sure it doesn't conflict with your existing OpenVSwitch bridges.
 ovsBridge: {{ .Values.ovs.bridgeName | quote }}

diff --git a/build/charts/antrea/crds/externalippool.yaml b/build/charts/antrea/crds/externalippool.yaml
@@ -133,6 +133,25 @@ spec:
                         oneOf:
                           - format: ipv4
                           - format: ipv6
+                subnetInfo:
+                  type: object
+                  required:
+                    - gateway
+                    - prefixLength
+                  properties:
+                    gateway:
+                      type: string
+                      oneOf:
+                        - format: ipv4
+                        - format: ipv6
+                    prefixLength:
+                      type: integer
+                      minimum: 1
+                      maximum: 127
+                    vlan:
+                      type: integer
+                      minimum: 0
+                      maximum: 4094
                 nodeSelector:
                   type: object
                   properties:

diff --git a/build/charts/antrea/templates/webhooks/validating/crdvalidator.yaml b/build/charts/antrea/templates/webhooks/validating/crdvalidator.yaml
@@ -116,9 +116,9 @@ webhooks:
         namespace: {{ .Release.Namespace }}
         path: "/validate/externalippool"
     rules:
-      - operations: ["UPDATE"]
+      - operations: ["CREATE", "UPDATE"]
         apiGroups: ["crd.antrea.io"]
-        apiVersions: ["v1alpha2"]
+        apiVersions: ["v1alpha2", "v1beta1"]
         resources: ["externalippools"]
         scope: "Cluster"
     admissionReviewVersions: ["v1", "v1beta1"]

diff --git a/build/yamls/antrea-aks.yml b/build/yamls/antrea-aks.yml
@@ -2654,6 +2654,25 @@ spec:
                         oneOf:
                           - format: ipv4
                           - format: ipv6
+                subnetInfo:
+                  type: object
+                  required:
+                    - gateway
+                    - prefixLength
+                  properties:
+                    gateway:
+                      type: string
+                      oneOf:
+                        - format: ipv4
+                        - format: ipv6
+                    prefixLength:
+                      type: integer
+                      minimum: 1
+                      maximum: 127
+                    vlan:
+                      type: integer
+                      minimum: 0
+                      maximum: 4094
                 nodeSelector:
                   type: object
                   properties:
@@ -5603,6 +5622,9 @@ data:
     # Enable Egress traffic shaping.
     #  EgressTrafficShaping: false
 
+    # Allow users to allocate Egress IPs from a separate subnet different from the default Node subnet.
+    #  EgressSeparateSubnet: false
+
     # Name of the OpenVSwitch bridge antrea-agent will create and use.
     # Make sure it doesn't conflict with your existing OpenVSwitch bridges.
     ovsBridge: "br-int"
@@ -6903,7 +6925,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: e59e0431902646d46cba490279184fea2bdd3c8b486b5a7b1d3ece9a91614634
+        checksum/config: 34f7e1dc5a957bd3106fae1e0b94a6a50336420be33a9378e02e795364c007e6
       labels:
         app: antrea
         component: antrea-agent
@@ -7141,7 +7163,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: e59e0431902646d46cba490279184fea2bdd3c8b486b5a7b1d3ece9a91614634
+        checksum/config: 34f7e1dc5a957bd3106fae1e0b94a6a50336420be33a9378e02e795364c007e6
       labels:
         app: antrea
         component: antrea-controller
@@ -7456,9 +7478,9 @@ webhooks:
         namespace: kube-system
         path: "/validate/externalippool"
     rules:
-      - operations: ["UPDATE"]
+      - operations: ["CREATE", "UPDATE"]
         apiGroups: ["crd.antrea.io"]
-        apiVersions: ["v1alpha2"]
+        apiVersions: ["v1alpha2", "v1beta1"]
         resources: ["externalippools"]
         scope: "Cluster"
     admissionReviewVersions: ["v1", "v1beta1"]

diff --git a/build/yamls/antrea-crds.yml b/build/yamls/antrea-crds.yml
@@ -2641,6 +2641,25 @@ spec:
                         oneOf:
                           - format: ipv4
                           - format: ipv6
+                subnetInfo:
+                  type: object
+                  required:
+                    - gateway
+                    - prefixLength
+                  properties:
+                    gateway:
+                      type: string
+                      oneOf:
+                        - format: ipv4
+                        - format: ipv6
+                    prefixLength:
+                      type: integer
+                      minimum: 1
+                      maximum: 127
+                    vlan:
+                      type: integer
+                      minimum: 0
+                      maximum: 4094
                 nodeSelector:
                   type: object
                   properties:

diff --git a/build/yamls/antrea-eks.yml b/build/yamls/antrea-eks.yml
@@ -2654,6 +2654,25 @@ spec:
                         oneOf:
                           - format: ipv4
                           - format: ipv6
+                subnetInfo:
+                  type: object
+                  required:
+                    - gateway
+                    - prefixLength
+                  properties:
+                    gateway:
+                      type: string
+                      oneOf:
+                        - format: ipv4
+                        - format: ipv6
+                    prefixLength:
+                      type: integer
+                      minimum: 1
+                      maximum: 127
+                    vlan:
+                      type: integer
+                      minimum: 0
+                      maximum: 4094
                 nodeSelector:
                   type: object
                   properties:
@@ -5603,6 +5622,9 @@ data:
     # Enable Egress traffic shaping.
     #  EgressTrafficShaping: false
 
+    # Allow users to allocate Egress IPs from a separate subnet different from the default Node subnet.
+    #  EgressSeparateSubnet: false
+
     # Name of the OpenVSwitch bridge antrea-agent will create and use.
     # Make sure it doesn't conflict with your existing OpenVSwitch bridges.
     ovsBridge: "br-int"
@@ -6903,7 +6925,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: e59e0431902646d46cba490279184fea2bdd3c8b486b5a7b1d3ece9a91614634
+        checksum/config: 34f7e1dc5a957bd3106fae1e0b94a6a50336420be33a9378e02e795364c007e6
       labels:
         app: antrea
         component: antrea-agent
@@ -7142,7 +7164,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: e59e0431902646d46cba490279184fea2bdd3c8b486b5a7b1d3ece9a91614634
+        checksum/config: 34f7e1dc5a957bd3106fae1e0b94a6a50336420be33a9378e02e795364c007e6
       labels:
         app: antrea
         component: antrea-controller
@@ -7457,9 +7479,9 @@ webhooks:
         namespace: kube-system
         path: "/validate/externalippool"
     rules:
-      - operations: ["UPDATE"]
+      - operations: ["CREATE", "UPDATE"]
         apiGroups: ["crd.antrea.io"]
-        apiVersions: ["v1alpha2"]
+        apiVersions: ["v1alpha2", "v1beta1"]
         resources: ["externalippools"]
         scope: "Cluster"
     admissionReviewVersions: ["v1", "v1beta1"]

diff --git a/build/yamls/antrea-gke.yml b/build/yamls/antrea-gke.yml
@@ -2654,6 +2654,25 @@ spec:
                         oneOf:
                           - format: ipv4
                           - format: ipv6
+                subnetInfo:
+                  type: object
+                  required:
+                    - gateway
+                    - prefixLength
+                  properties:
+                    gateway:
+                      type: string
+                      oneOf:
+                        - format: ipv4
+                        - format: ipv6
+                    prefixLength:
+                      type: integer
+                      minimum: 1
+                      maximum: 127
+                    vlan:
+                      type: integer
+                      minimum: 0
+                      maximum: 4094
                 nodeSelector:
                   type: object
                   properties:
@@ -5603,6 +5622,9 @@ data:
     # Enable Egress traffic shaping.
     #  EgressTrafficShaping: false
 
+    # Allow users to allocate Egress IPs from a separate subnet different from the default Node subnet.
+    #  EgressSeparateSubnet: false
+
     # Name of the OpenVSwitch bridge antrea-agent will create and use.
     # Make sure it doesn't conflict with your existing OpenVSwitch bridges.
     ovsBridge: "br-int"
@@ -6903,7 +6925,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 3b1758664de8044af1aa7454c64bd1a4911750e562e1ae9375c9c16a335a469d
+        checksum/config: 791d3c52ac84fd8a6db2a6fee50ee2dce7e04f49c722bf8e2967764f39e6866e
       labels:
         app: antrea
         component: antrea-agent
@@ -7139,7 +7161,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 3b1758664de8044af1aa7454c64bd1a4911750e562e1ae9375c9c16a335a469d
+        checksum/config: 791d3c52ac84fd8a6db2a6fee50ee2dce7e04f49c722bf8e2967764f39e6866e
       labels:
         app: antrea
         component: antrea-controller
@@ -7454,9 +7476,9 @@ webhooks:
         namespace: kube-system
         path: "/validate/externalippool"
     rules:
-      - operations: ["UPDATE"]
+      - operations: ["CREATE", "UPDATE"]
         apiGroups: ["crd.antrea.io"]
-        apiVersions: ["v1alpha2"]
+        apiVersions: ["v1alpha2", "v1beta1"]
         resources: ["externalippools"]
         scope: "Cluster"
     admissionReviewVersions: ["v1", "v1beta1"]

diff --git a/build/yamls/antrea-ipsec.yml b/build/yamls/antrea-ipsec.yml
@@ -2654,6 +2654,25 @@ spec:
                         oneOf:
                           - format: ipv4
                           - format: ipv6
+                subnetInfo:
+                  type: object
+                  required:
+                    - gateway
+                    - prefixLength
+                  properties:
+                    gateway:
+                      type: string
+                      oneOf:
+                        - format: ipv4
+                        - format: ipv6
+                    prefixLength:
+                      type: integer
+                      minimum: 1
+                      maximum: 127
+                    vlan:
+                      type: integer
+                      minimum: 0
+                      maximum: 4094
                 nodeSelector:
                   type: object
                   properties:
@@ -5616,6 +5635,9 @@ data:
     # Enable Egress traffic shaping.
     #  EgressTrafficShaping: false
 
+    # Allow users to allocate Egress IPs from a separate subnet different from the default Node subnet.
+    #  EgressSeparateSubnet: false
+
     # Name of the OpenVSwitch bridge antrea-agent will create and use.
     # Make sure it doesn't conflict with your existing OpenVSwitch bridges.
     ovsBridge: "br-int"
@@ -6916,7 +6938,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: a34de3efa658ac40c9bde28e08832dd897259fdcf639beab9d4e47531d7da948
+        checksum/config: 81a9d6f9ca6306467ca8a241709027336626dfaad557d3cbb5cfa7028b0af9a5
         checksum/ipsec-secret: d0eb9c52d0cd4311b6d252a951126bf9bea27ec05590bed8a394f0f792dcb2a4
       labels:
         app: antrea
@@ -7198,7 +7220,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: a34de3efa658ac40c9bde28e08832dd897259fdcf639beab9d4e47531d7da948
+        checksum/config: 81a9d6f9ca6306467ca8a241709027336626dfaad557d3cbb5cfa7028b0af9a5
       labels:
         app: antrea
         component: antrea-controller
@@ -7513,9 +7535,9 @@ webhooks:
         namespace: kube-system
         path: "/validate/externalippool"
     rules:
-      - operations: ["UPDATE"]
+      - operations: ["CREATE", "UPDATE"]
         apiGroups: ["crd.antrea.io"]
-        apiVersions: ["v1alpha2"]
+        apiVersions: ["v1alpha2", "v1beta1"]
         resources: ["externalippools"]
         scope: "Cluster"
     admissionReviewVersions: ["v1", "v1beta1"]