L7 Network Flow Export support in Antrea (antrea-io#5218)

L7 Network flow export enables Layer 7 flow export as we did for the l3/l4 earlier. This feature enables the user to export L7 protocol information using: - Pod or namespace Annotations Annotation key used is "visibility.antrea.io/l7-export" and the value is the direction for which the flow export is required, which could be (ingress/egress/both). Based of the annotation or the NP, the fields ("appProtocolName", "httpVals") are populated and exported. Signed-off-by: Tushar Tathgur <[email protected]>
antoninbas · Jan 24, 2024 · f49a4ea · f49a4ea
1 parent 18dc013
commit f49a4ea
Show file tree

Hide file tree

Showing 59 changed files with 1,991 additions and 184 deletions.
diff --git a/build/charts/antrea/conf/antrea-agent.conf b/build/charts/antrea/conf/antrea-agent.conf
@@ -85,6 +85,9 @@ featureGates:
 # Allow users to apply ClusterNetworkPolicy to Kubernetes Nodes.
 {{- include "featureGate" (dict "featureGates" .Values.featureGates "name" "NodeNetworkPolicy" "default" false) }}
 
+# Enable L7FlowExporter on Pods and Namespaces to export the application layer flows such as HTTP flows.
+{{- include "featureGate" (dict "featureGates" .Values.featureGates "name" "L7FlowExporter" "default" false) }}
+
 # Name of the OpenVSwitch bridge antrea-agent will create and use.
 # Make sure it doesn't conflict with your existing OpenVSwitch bridges.
 ovsBridge: {{ .Values.ovs.bridgeName | quote }}

diff --git a/build/yamls/antrea-aks.yml b/build/yamls/antrea-aks.yml
@@ -5628,6 +5628,9 @@ data:
     # Allow users to apply ClusterNetworkPolicy to Kubernetes Nodes.
     #  NodeNetworkPolicy: false
 
+    # Enable L7FlowExporter on Pods and Namespaces to export the application layer flows such as HTTP flows.
+    #  L7FlowExporter: false
+
     # Name of the OpenVSwitch bridge antrea-agent will create and use.
     # Make sure it doesn't conflict with your existing OpenVSwitch bridges.
     ovsBridge: "br-int"
@@ -6928,7 +6931,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: f4ad8910666191c02982d1b7b202e3c4bd20fb4a8179dcb5696119f3b1490a72
+        checksum/config: 30843b57762c91dfcffb560917191e3bc7e662c06552759bac2a173bc060b82c
       labels:
         app: antrea
         component: antrea-agent
@@ -7166,7 +7169,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: f4ad8910666191c02982d1b7b202e3c4bd20fb4a8179dcb5696119f3b1490a72
+        checksum/config: 30843b57762c91dfcffb560917191e3bc7e662c06552759bac2a173bc060b82c
       labels:
         app: antrea
         component: antrea-controller

diff --git a/build/yamls/antrea-eks.yml b/build/yamls/antrea-eks.yml
@@ -5628,6 +5628,9 @@ data:
     # Allow users to apply ClusterNetworkPolicy to Kubernetes Nodes.
     #  NodeNetworkPolicy: false
 
+    # Enable L7FlowExporter on Pods and Namespaces to export the application layer flows such as HTTP flows.
+    #  L7FlowExporter: false
+
     # Name of the OpenVSwitch bridge antrea-agent will create and use.
     # Make sure it doesn't conflict with your existing OpenVSwitch bridges.
     ovsBridge: "br-int"
@@ -6928,7 +6931,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: f4ad8910666191c02982d1b7b202e3c4bd20fb4a8179dcb5696119f3b1490a72
+        checksum/config: 30843b57762c91dfcffb560917191e3bc7e662c06552759bac2a173bc060b82c
       labels:
         app: antrea
         component: antrea-agent
@@ -7167,7 +7170,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: f4ad8910666191c02982d1b7b202e3c4bd20fb4a8179dcb5696119f3b1490a72
+        checksum/config: 30843b57762c91dfcffb560917191e3bc7e662c06552759bac2a173bc060b82c
       labels:
         app: antrea
         component: antrea-controller

diff --git a/build/yamls/antrea-gke.yml b/build/yamls/antrea-gke.yml
@@ -5628,6 +5628,9 @@ data:
     # Allow users to apply ClusterNetworkPolicy to Kubernetes Nodes.
     #  NodeNetworkPolicy: false
 
+    # Enable L7FlowExporter on Pods and Namespaces to export the application layer flows such as HTTP flows.
+    #  L7FlowExporter: false
+
     # Name of the OpenVSwitch bridge antrea-agent will create and use.
     # Make sure it doesn't conflict with your existing OpenVSwitch bridges.
     ovsBridge: "br-int"
@@ -6928,7 +6931,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: a54768c79d693083be554386f268c93bbbd0fdf5b334edd9aff31c13151c4e29
+        checksum/config: d5cdb5356795c44a69c66fad1b4d67f7c00cdcbe837f3b3b50260e4d9dfd1e7e
       labels:
         app: antrea
         component: antrea-agent
@@ -7164,7 +7167,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: a54768c79d693083be554386f268c93bbbd0fdf5b334edd9aff31c13151c4e29
+        checksum/config: d5cdb5356795c44a69c66fad1b4d67f7c00cdcbe837f3b3b50260e4d9dfd1e7e
       labels:
         app: antrea
         component: antrea-controller

diff --git a/build/yamls/antrea-ipsec.yml b/build/yamls/antrea-ipsec.yml
@@ -5641,6 +5641,9 @@ data:
     # Allow users to apply ClusterNetworkPolicy to Kubernetes Nodes.
     #  NodeNetworkPolicy: false
 
+    # Enable L7FlowExporter on Pods and Namespaces to export the application layer flows such as HTTP flows.
+    #  L7FlowExporter: false
+
     # Name of the OpenVSwitch bridge antrea-agent will create and use.
     # Make sure it doesn't conflict with your existing OpenVSwitch bridges.
     ovsBridge: "br-int"
@@ -6941,7 +6944,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 7ce7d85bc08079d1cef3b1d44f31e2139961f9ae49f71d79ff3b28e7e9ad6325
+        checksum/config: 50f2864cf09e4732327b963130bd59a9fc06c560784b161c94e813c000367615
         checksum/ipsec-secret: d0eb9c52d0cd4311b6d252a951126bf9bea27ec05590bed8a394f0f792dcb2a4
       labels:
         app: antrea
@@ -7223,7 +7226,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 7ce7d85bc08079d1cef3b1d44f31e2139961f9ae49f71d79ff3b28e7e9ad6325
+        checksum/config: 50f2864cf09e4732327b963130bd59a9fc06c560784b161c94e813c000367615
       labels:
         app: antrea
         component: antrea-controller

diff --git a/build/yamls/antrea.yml b/build/yamls/antrea.yml
@@ -5628,6 +5628,9 @@ data:
     # Allow users to apply ClusterNetworkPolicy to Kubernetes Nodes.
     #  NodeNetworkPolicy: false
 
+    # Enable L7FlowExporter on Pods and Namespaces to export the application layer flows such as HTTP flows.
+    #  L7FlowExporter: false
+
     # Name of the OpenVSwitch bridge antrea-agent will create and use.
     # Make sure it doesn't conflict with your existing OpenVSwitch bridges.
     ovsBridge: "br-int"
@@ -6928,7 +6931,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 290f0c748863a7dad1e9d53d62c74f8108a44c5cc803306d351c108062cc1378
+        checksum/config: ac3c14eed7ca0dc28bf2d659cd2c4e4a39d55278fb9a8759c30ea12eff89e518
       labels:
         app: antrea
         component: antrea-agent
@@ -7164,7 +7167,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 290f0c748863a7dad1e9d53d62c74f8108a44c5cc803306d351c108062cc1378
+        checksum/config: ac3c14eed7ca0dc28bf2d659cd2c4e4a39d55278fb9a8759c30ea12eff89e518
       labels:
         app: antrea
         component: antrea-controller

diff --git a/ci/kind/test-e2e-kind.sh b/ci/kind/test-e2e-kind.sh
@@ -205,7 +205,7 @@ if $multicast; then
     manifest_args="$manifest_args --multicast"
 fi
 if $flow_visibility; then
-    manifest_args="$manifest_args --feature-gates FlowExporter=true --extra-helm-values-file $FLOW_VISIBILITY_HELM_VALUES"
+    manifest_args="$manifest_args --feature-gates FlowExporter=true,L7FlowExporter=true --extra-helm-values-file $FLOW_VISIBILITY_HELM_VALUES"
 fi
 
 COMMON_IMAGES_LIST=("registry.k8s.io/e2e-test-images/agnhost:2.29" \

diff --git a/cmd/antrea-agent/agent.go b/cmd/antrea-agent/agent.go
@@ -39,7 +39,9 @@ import (
 	"antrea.io/antrea/pkg/agent/config"
 	"antrea.io/antrea/pkg/agent/controller/egress"
 	"antrea.io/antrea/pkg/agent/controller/ipseccertificate"
+	"antrea.io/antrea/pkg/agent/controller/l7flowexporter"
 	"antrea.io/antrea/pkg/agent/controller/networkpolicy"
+	"antrea.io/antrea/pkg/agent/controller/networkpolicy/l7engine"
 	"antrea.io/antrea/pkg/agent/controller/noderoute"
 	"antrea.io/antrea/pkg/agent/controller/serviceexternalip"
 	"antrea.io/antrea/pkg/agent/controller/traceflow"
@@ -141,6 +143,7 @@ func run(o *Options) error {
 	enableBridgingMode := enableAntreaIPAM && o.config.EnableBridgingMode
 	l7NetworkPolicyEnabled := features.DefaultFeatureGate.Enabled(features.L7NetworkPolicy)
 	nodeNetworkPolicyEnabled := features.DefaultFeatureGate.Enabled(features.NodeNetworkPolicy)
+	l7FlowExporterEnabled := features.DefaultFeatureGate.Enabled(features.L7FlowExporter)
 	enableMulticlusterGW := features.DefaultFeatureGate.Enabled(features.Multicluster) && o.config.Multicluster.EnableGateway
 	enableMulticlusterNP := features.DefaultFeatureGate.Enabled(features.Multicluster) && o.config.Multicluster.EnableStretchedNetworkPolicy
 	enableFlowExporter := features.DefaultFeatureGate.Enabled(features.FlowExporter) && o.config.FlowExporter.Enable
@@ -170,6 +173,7 @@ func run(o *Options) error {
 		connectUplinkToBridge,
 		multicastEnabled,
 		features.DefaultFeatureGate.Enabled(features.TrafficControl),
+		l7FlowExporterEnabled,
 		enableMulticlusterGW,
 		groupIDAllocator,
 		*o.config.EnablePrometheusMetrics,
@@ -292,7 +296,8 @@ func run(o *Options) error {
 		o.config.ExternalNode.ExternalNodeNamespace,
 		connectUplinkToBridge,
 		o.enableAntreaProxy,
-		l7NetworkPolicyEnabled)
+		l7NetworkPolicyEnabled,
+		l7FlowExporterEnabled)
 	err = agentInitializer.Initialize()
 	if err != nil {
 		return fmt.Errorf("error initializing agent: %v", err)
@@ -466,6 +471,10 @@ func run(o *Options) error {
 	if o.nodeType == config.ExternalNode {
 		nodeKey = k8s.NamespacedName(o.config.ExternalNode.ExternalNodeNamespace, nodeKey)
 	}
+	var l7Reconciler *l7engine.Reconciler
+	if l7NetworkPolicyEnabled || l7FlowExporterEnabled {
+		l7Reconciler = l7engine.NewReconciler()
+	}
 	networkPolicyController, err := networkpolicy.NewNetworkPolicyController(
 		antreaClientProvider,
 		ofClient,
@@ -493,10 +502,22 @@ func run(o *Options) error {
 		tunPort,
 		nodeConfig,
 		podNetworkWait,
+		l7Reconciler,
 	)
 	if err != nil {
 		return fmt.Errorf("error creating new NetworkPolicy controller: %v", err)
 	}
+	var l7FlowExporterController *l7flowexporter.L7FlowExporterController
+	if l7FlowExporterEnabled {
+		l7FlowExporterController = l7flowexporter.NewL7FlowExporterController(
+			ofClient,
+			ifaceStore,
+			localPodInformer.Get(),
+			namespaceInformer,
+			l7Reconciler,
+		)
+		go l7FlowExporterController.Run(stopCh)
+	}
 
 	var egressController *egress.EgressController
 
@@ -650,7 +671,9 @@ func run(o *Options) error {
 			o.enableAntreaProxy,
 			networkPolicyController,
 			flowExporterOptions,
-			egressController)
+			egressController,
+			l7FlowExporterController,
+			l7FlowExporterEnabled)
 		if err != nil {
 			return fmt.Errorf("error when creating IPFIX flow exporter: %v", err)
 		}

diff --git a/docs/antrea-l7-network-policy.md b/docs/antrea-l7-network-policy.md
@@ -72,7 +72,7 @@ forwarded to an application-aware engine for protocol detection and rule enforce
 the layer 7 criteria is also matched, otherwise it will be dropped. Therefore, any rules after a layer 7 rule will not
 be enforced for the traffic that match the layer 7 rule's layer 3/4 criteria.
 
-As of now, the only supported layer 7 protocol is HTTP. More protocols will be supported in the near future, and we
+As of now, the only supported layer 7 protocol is HTTP. Support for more protocols may be added in the future and we
 welcome feature requests for protocols that you are interested in.
 
 ### HTTP

diff --git a/docs/feature-gates.md b/docs/feature-gates.md
@@ -58,6 +58,7 @@ edit the Agent configuration in the
 | `EgressTrafficShaping`        | Agent              | `false` | Alpha | v1.14         | N/A          | N/A        | Yes                | OVS meters should be supported                |
 | `EgressSeparateSubnet`        | Agent              | `false` | Alpha | v1.15         | N/A          | N/A        | No                 |                                               |
 | `NodeNetworkPolicy`           | Agent              | `false` | Alpha | v1.15         | N/A          | N/A        | Yes                |                                               |
+| `L7FlowExporter`              | Agent              | `false` | Alpha | v1.15         | N/A          | N/A        | Yes                |                                               |
 
 ## Description and Requirements of Features
 
@@ -428,3 +429,12 @@ to be supported in the datapath.
 
 `EgressSeparateSubnet` allows users to allocate Egress IPs from a different subnet from the default Node subnet.
 Refer to this [document](egress.md#subnetinfo) for more information.
+
+### L7FlowExporter
+
+`L7FlowExporter` enables users to export application-layer flow data using Pod or Namespace annotations.
+Refer to this [document](network-flow-visibility.md#l7-visibility) for more information.
+
+#### Requirements for this Feature
+
+- Linux Nodes only.
diff --git a/docs/network-flow-visibility.md b/docs/network-flow-visibility.md
@@ -35,6 +35,9 @@
     - [Output Flow Records](#output-flow-records)
   - [Grafana Flow Collector (migrated)](#grafana-flow-collector-migrated)
   - [ELK Flow Collector (removed)](#elk-flow-collector-removed)
+- [Layer 7 Network Flow Exporter](#layer-7-network-flow-exporter)
+  - [Prerequisites](#prerequisites)
+  - [Usage](#usage)
 <!-- /toc -->
 
 ## Overview
@@ -610,3 +613,59 @@ and other Theia features, please refer to the
 **Starting with Antrea v1.7, support for the ELK Flow Collector has been removed.**
 Please consider using the [Grafana Flow Collector](#grafana-flow-collector-migrated)
 instead, which is actively maintained.
+
+## Layer 7 Network Flow Exporter
+
+In addition to layer 4 network visibility, Antrea adds layer 7 network flow
+export.
+
+### Prerequisites
+
+To achieve L7 (Layer 7) network flow export, the `L7FlowExporter` feature gate
+must be enabled.
+
+### Usage
+
+To export layer 7 flows of a Pod or a Namespace, user can annotate Pods or
+Namespaces with the annotation key `visibility.antrea.io/l7-export` and set the
+value to indicate the traffic flow direction, which can be `ingress`, `egress`
+or `both`.
+
+For example, to enable L7 flow export in the ingress direction on
+Pod test-pod in the default Namespace, you can use:
+
+```bash
+kubectl annotate pod test-pod visibility.antrea.io/l7-export=ingress
+```
+
+Based on the annotation, Flow Exporter will export the L7 flow data to the
+Flow Aggregator or configured IPFix collector using the fields `appProtocolName`
+and `httpVals`.
+
+* `appProtocolName` field is used to indicate the application layer protocol
+name (e.g. http) and it will be empty if application layer data is not exported.
+* `httpVals` stores a serialized JSON dictionary with every HTTP request for
+a connection mapped to a unique transaction ID. This format lets us group all
+the HTTP transactions pertaining to the same connection, into the same exported
+record.
+
+An example of `httpVals` is :
+
+`"{\"0\":{\"hostname\":\"10.10.0.1\",\"url\":\"/public/\",\"http_user_agent\":\"curl/7.74.0\",\"http_content_type\":\"text/html\",\"http_method\":\"GET\",\"protocol\":\"HTTP/1.1\",\"status\":200,\"length\":153}}"`
+
+HTTP fields in the `httpVals` are:
+
+| Http field        | Description                                            |
+|-------------------|--------------------------------------------------------|
+| hostname          | IP address of the sender                               |
+| URL               | url requested on the server                            |
+| http_user_agent   | application used for HTTP                              |
+| http_content_type | type of content being returned by the server           |
+| http_method       | HTTP method used for the request                       |
+| protocol          | HTTP protocol version used for the request or response |
+| status            | HTTP status code                                       |
+| length            | size of the response body                              |
+
+As of now, the only supported layer 7 protocol is `HTTP1.1`. Support for more
+protocols may be added in the future. Antrea supports L7FlowExporter feature only
+on Linux Nodes.
diff --git a/pkg/agent/agent.go b/pkg/agent/agent.go
@@ -119,6 +119,7 @@ type Initializer struct {
 	serviceConfig         *config.ServiceConfig
 	l7NetworkPolicyConfig *config.L7NetworkPolicyConfig
 	enableL7NetworkPolicy bool
+	enableL7FlowExporter  bool
 	connectUplinkToBridge bool
 	enableAntreaProxy     bool
 	// podNetworkWait should be decremented once the Node's network is ready.
@@ -151,6 +152,7 @@ func NewInitializer(
 	connectUplinkToBridge bool,
 	enableAntreaProxy bool,
 	enableL7NetworkPolicy bool,
+	enableL7FlowExporter bool,
 ) *Initializer {
 	return &Initializer{
 		ovsBridgeClient:       ovsBridgeClient,
@@ -175,6 +177,7 @@ func NewInitializer(
 		connectUplinkToBridge: connectUplinkToBridge,
 		enableAntreaProxy:     enableAntreaProxy,
 		enableL7NetworkPolicy: enableL7NetworkPolicy,
+		enableL7FlowExporter:  enableL7FlowExporter,
 	}
 }
 
@@ -423,9 +426,9 @@ func (i *Initializer) Initialize() error {
 		return err
 	}
 
-	if i.enableL7NetworkPolicy {
-		// prepareL7NetworkPolicyInterfaces must be executed after setupOVSBridge since it requires interfaceStore.
-		if err := i.prepareL7NetworkPolicyInterfaces(); err != nil {
+	if i.enableL7NetworkPolicy || i.enableL7FlowExporter {
+		// prepareL7EngineInterfaces must be executed after setupOVSBridge since it requires interfaceStore.
+		if err := i.prepareL7EngineInterfaces(); err != nil {
 			return err
 		}
 	}