traffic rate for multus nic

fanriming · May 20, 2021 · 87a5c70 · 87a5c70
1 parent 19c0fd5
commit 87a5c70
Show file tree

Hide file tree

Showing 5 changed files with 51 additions and 11 deletions.
diff --git a/pkg/daemon/controller.go b/pkg/daemon/controller.go
@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"net"
 	"os/exec"
+	"strings"
 	"time"
 
 	"k8s.io/client-go/kubernetes/scheme"
@@ -292,9 +293,18 @@ func (c *Controller) enqueuePod(old, new interface{}) {
 	oldPod := old.(*v1.Pod)
 	newPod := new.(*v1.Pod)
 
-	if oldPod.Annotations[util.IngressRateAnnotation] != newPod.Annotations[util.IngressRateAnnotation] ||
-		oldPod.Annotations[util.EgressRateAnnotation] != newPod.Annotations[util.EgressRateAnnotation] ||
-		oldPod.Annotations[util.VlanIdAnnotation] != newPod.Annotations[util.VlanIdAnnotation] {
+	trafficeRateChange := false
+	for k, _ := range newPod.Annotations {
+		if !strings.HasSuffix(k, util.IngressRateAnnotation) && !strings.HasSuffix(k, util.EgressRateAnnotation) {
+			continue
+		}
+		if oldPod.Annotations[k] != newPod.Annotations[k] {
+			trafficeRateChange = true
+			break
+		}
+	}
+
+	if trafficeRateChange || oldPod.Annotations[util.VlanIdAnnotation] != newPod.Annotations[util.VlanIdAnnotation] {
 		var key string
 		var err error
 		if key, err = cache.MetaNamespaceKeyFunc(new); err != nil {
@@ -364,7 +374,26 @@ func (c *Controller) handlePod(key string) error {
 		return err
 	}
 
-	return ovs.SetInterfaceBandwidth(fmt.Sprintf("%s.%s", pod.Name, pod.Namespace), pod.Annotations[util.EgressRateAnnotation], pod.Annotations[util.IngressRateAnnotation])
+	// set default nic bandwidth
+	ifaceID := ovs.PodNameToPortName(pod.Name, pod.Namespace, util.OvnProvider)
+	err = ovs.SetInterfaceBandwidth(pod.Name, pod.Namespace, ifaceID, pod.Annotations[util.EgressRateAnnotation], pod.Annotations[util.IngressRateAnnotation])
+	if err != nil {
+		return err
+	}
+
+	// set multis-nic bandwidth
+	attachNets, err := util.ParsePodNetworkAnnotation(pod.Annotations[util.AttachmentNetworkAnnotation], pod.Namespace)
+	for _, multiNet := range attachNets {
+		provider := fmt.Sprintf("%s.%s.ovn", multiNet.Name, multiNet.Namespace)
+		if pod.Annotations[fmt.Sprintf(util.AllocatedAnnotationTemplate, provider)] == "true" {
+			ifaceID = ovs.PodNameToPortName(pod.Name, pod.Namespace, provider)
+			err = ovs.SetInterfaceBandwidth(pod.Name, pod.Namespace, ifaceID, pod.Annotations[fmt.Sprintf(util.IngressRateAnnotationTemplate, provider)], pod.Annotations[fmt.Sprintf(util.EgressRateAnnotationTemplate, provider)])
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
 }
 
 // Run starts controller

diff --git a/pkg/daemon/gateway.go b/pkg/daemon/gateway.go
@@ -144,7 +144,7 @@ func (c *Controller) setGatewayBandwidth() error {
 	}
 	ingress, egress := node.Annotations[util.IngressRateAnnotation], node.Annotations[util.EgressRateAnnotation]
 	ifaceId := fmt.Sprintf("node-%s", c.config.NodeName)
-	return ovs.SetInterfaceBandwidth(ifaceId, egress, ingress)
+	return ovs.SetInterfaceBandwidth("", "", ifaceId, egress, ingress)
 }
 
 func (c *Controller) setICGateway() error {

diff --git a/pkg/daemon/ovs.go b/pkg/daemon/ovs.go
@@ -56,7 +56,7 @@ func (csh cniServerHandler) configureNic(podName, podNamespace, provider, netns,
 	if err = configureHostNic(hostNicName, vlanID); err != nil {
 		return err
 	}
-	if err = ovs.SetInterfaceBandwidth(fmt.Sprintf("%s.%s", podName, podNamespace), ingress, egress); err != nil {
+	if err = ovs.SetInterfaceBandwidth(podName, podNamespace, ifaceID, ingress, egress); err != nil {
 		return err
 	}
 

diff --git a/pkg/ovs/ovs-vsctl.go b/pkg/ovs/ovs-vsctl.go
@@ -2,11 +2,13 @@ package ovs
 
 import (
 	"fmt"
-	"k8s.io/klog"
 	"os/exec"
 	"strconv"
 	"strings"
 	"time"
+
+	"github.com/alauda/kube-ovn/pkg/util"
+	"k8s.io/klog"
 )
 
 // Glory belongs to openvswitch/ovn-kubernetes
@@ -80,6 +82,12 @@ func ClearPodBandwidth(podName, podNamespace string) error {
 	if err != nil {
 		return err
 	}
+	qosListByPod, err := ovsFind("qos", "_uuid", fmt.Sprintf(`external-ids:pod="%s/%s"`, podNamespace, podName))
+	if err != nil {
+		return err
+	}
+	qosList = append(qosList, qosListByPod...)
+	qosList = util.UniqString(qosList)
 	for _, qos := range qosList {
 		if err := ovsDestroy("qos", qos); err != nil {
 			return err
@@ -89,7 +97,7 @@ func ClearPodBandwidth(podName, podNamespace string) error {
 }
 
 // SetInterfaceBandwidth set ingress/egress qos for given pod
-func SetInterfaceBandwidth(iface, ingress, egress string) error {
+func SetInterfaceBandwidth(podName, podNamespace, iface, ingress, egress string) error {
 	ingressMPS, _ := strconv.Atoi(ingress)
 	ingressKPS := ingressMPS * 1000
 	interfaceList, err := ovsFind("interface", "name", fmt.Sprintf("external-ids:iface-id=%s", iface))
@@ -113,7 +121,10 @@ func SetInterfaceBandwidth(iface, ingress, egress string) error {
 		}
 		if egressBPS > 0 {
 			if len(qosList) == 0 {
-				qos, err := ovsCreate("qos", "type=linux-htb", fmt.Sprintf("other-config:max-rate=%d", egressBPS), fmt.Sprintf("external-ids:iface-id=%s", iface))
+				qos, err := ovsCreate("qos", "type=linux-htb",
+					fmt.Sprintf("other-config:max-rate=%d", egressBPS),
+					fmt.Sprintf("external-ids:iface-id=%s", iface),
+					fmt.Sprintf("external-ids:pod=%s/%s", podNamespace, podName))
 				if err != nil {
 					return err
 				}

diff --git a/pkg/util/const.go b/pkg/util/const.go
@@ -28,8 +28,8 @@ const (
 	IpPoolAnnotationTemplate        = "%s.kubernetes.io/ip_pool"
 	LogicalSwitchAnnotationTemplate = "%s.kubernetes.io/logical_switch"
 	VlanIdAnnotationTemplate        = "%s.kubernetes.io/vlan_id"
-	IngressRateAnnotationTemplate   = "%s.ovn.kubernetes.io/ingress_rate"
-	EgressRateAnnotationTemplate    = "%s.ovn.kubernetes.io/egress_rate"
+	IngressRateAnnotationTemplate   = "%s.kubernetes.io/ingress_rate"
+	EgressRateAnnotationTemplate    = "%s.kubernetes.io/egress_rate"
 
 	ExcludeIpsAnnotation = "ovn.kubernetes.io/exclude_ips"