spidermultusconfig: It's able to configure bandwidth for sriov config

spidernet-io · Nov 23, 2023 · c470c91 · c470c91
1 parent d8e42e9
commit c470c91
Show file tree

Hide file tree

Showing 10 changed files with 335 additions and 5 deletions.
diff --git a/charts/spiderpool/crds/spiderpool.spidernet.io_spidermultusconfigs.yaml b/charts/spiderpool/crds/spiderpool.spidernet.io_spidermultusconfigs.yaml
@@ -241,6 +241,12 @@ spec:
                           type: string
                         type: array
                     type: object
+                  maxTxRateMbps:
+                    minimum: 0
+                    type: integer
+                  minTxRateMbps:
+                    minimum: 0
+                    type: integer
                   resourceName:
                     type: string
                   vlanID:

diff --git a/docs/reference/crd-spidermultusconfig.md b/docs/reference/crd-spidermultusconfig.md
@@ -85,6 +85,8 @@ This is the SpiderReservedIP spec for users to configure.
 |--------------|-------------------------------------------------------------------------------------------|----------------------------------------------------------------|------------|
 | resourceName | this property will create an annotation for Multus net-attach-def to cooperate with SRIOV | string                                                         | required   |
 | vlanID       | vlan ID                                                                                   | int                                                            | optional   |
+| minTxRate    | change the allowed minimum transmit bandwidth, in Mbps, for the VF. Setting this to 0 disables rate limiting. The min_tx_rate value should be <= max_tx_rate. Support of this feature depends on NICs and drivers | int | optional |
+| maxTxRate    | change the allowed maximum transmit bandwidth, in Mbps, for the VF. Setting this to 0 disables rate limiting | int | optional |
 | ippools      | the default IPPools in your CNI configurations                                            | [SpiderpoolPools](./crd-spidermultusconfig.md#SpiderpoolPools) | optional   |
 | enableRdma   | enable rdma chain cni to isolate the rdma device                                          | bool                                                           | optional   |
 

diff --git a/docs/usage/spider-multus-config-zh_CN.md b/docs/usage/spider-multus-config-zh_CN.md
@@ -162,6 +162,145 @@ spec:
   config: '{"cniVersion":"0.3.1","name":"ipvlan-ens192","plugins":[{"type":"ipvlan","master":"ens192","ipam":{"type":"spiderpool"}},{"type":"coordinator"}]}'
 ```
 
+### 创建 Sriov 配置
+
+如下是创建 Sriov SpiderMultusConfig 配置的示例：
+
+- 一个基础的例子
+
+```bash
+cat <<EOF | kubectl apply -f -
+apiVersion: spiderpool.spidernet.io/v2beta1
+kind: SpiderMultusConfig
+metadata:
+  name: sriov-demo
+  namespace: kube-system
+spec:
+  cniType: sriov
+  enableCoordinator: true
+  sriov:
+    resourceName: spidernet.io/sriov_netdeivce
+    vlanID: 100
+EOF
+```
+
+创建后，查看对应的 Multus NetworkAttachmentDefinition CR:
+
+```shell
+~# kubectl get network-attachment-definitions.k8s.cni.cncf.io -n kube-system sriov-demo -o yaml
+apiVersion: k8s.cni.cncf.io/v1
+kind: NetworkAttachmentDefinition
+metadata:
+  name: sriov-demo
+  namespace: kube-system
+  annotations:
+    k8s.v1.cni.cncf.io/resourceName: spidernet.io/sriov_netdeivce 
+  ownerReferences:
+  - apiVersion: spiderpool.spidernet.io/v2beta1
+    blockOwnerDeletion: true
+    controller: true
+    kind: SpiderMultusConfig
+    name: sriov-demo
+    uid: b08ce054-1ae8-414a-b37c-7fd6988b1b8e
+  resourceVersion: "153002297"
+  uid: 4413e1fa-ce15-4acf-bce8-48b5028c0568
+spec:
+  config: '{"cniVersion":"0.3.1","name":"sriov-demo","plugins":[{"vlan":100,"type":"sriov","ipam":{"type":"spiderpool"}},{"type":"coordinator"}]}'
+```
+
+更多信息可参考 [Sriov-cni 使用](./install/underlay/get-started-sriov-zh_CN.md)
+
+- 配置启用 RDMA 功能
+
+```bash
+cat <<EOF | kubectl apply -f -
+apiVersion: spiderpool.spidernet.io/v2beta1
+kind: SpiderMultusConfig
+metadata:
+  name: sriov-rdma
+  namespace: kube-system
+spec:
+  cniType: sriov
+  enableCoordinator: true
+  sriov:
+    enableRdma: true
+    resourceName: spidernet.io/sriov_netdeivce
+    vlanID: 100
+EOF
+```
+
+创建后，查看对应的 Multus NetworkAttachmentDefinition CR:
+
+```shell
+~# kubectl get network-attachment-definitions.k8s.cni.cncf.io -n kube-system sriov-rdma -o yaml
+apiVersion: k8s.cni.cncf.io/v1
+kind: NetworkAttachmentDefinition
+metadata:
+  name: sriov-rdma
+  namespace: kube-system
+  annotations:
+    k8s.v1.cni.cncf.io/resourceName: spidernet.io/sriov_netdeivce 
+  ownerReferences:
+  - apiVersion: spiderpool.spidernet.io/v2beta1
+    blockOwnerDeletion: true
+    controller: true
+    kind: SpiderMultusConfig
+    name: sriov-rdma
+    uid: b08ce054-1ae8-414a-b37c-7fd6988b1b8e
+  resourceVersion: "153002297"
+  uid: 4413e1fa-ce15-4acf-bce8-48b5028c0568
+spec:
+  config: '{"cniVersion":"0.3.1","name":"sriov-rdma","plugins":[{"vlan":100,"type":"sriov","ipam":{"type":"spiderpool"}},{"type":"rdma"},{"type":"coordinator"}]}'
+```
+
+更多信息可参考 [Sriov-rdma 使用](rdma-zh_CN.md)
+
+- 配置 Sriov 网络带宽
+
+我们可通过 SpiderMultusConfig 配置 Sriov 的网络带宽:
+
+```bash
+cat <<EOF | kubectl apply -f -
+apiVersion: spiderpool.spidernet.io/v2beta1
+kind: SpiderMultusConfig
+metadata:
+  name: sriov-bandwidth
+  namespace: kube-system
+spec:
+  cniType: sriov
+  enableCoordinator: true
+  sriov:
+    resourceName: spidernet.io/sriov_netdeivce
+    vlanID: 100
+    minTxRateMbps: 100
+    MaxTxRateMbps: 1000
+EOF
+```
+
+> `minTxRateMbps` 和 `MaxTxRateMbps` 配置此 CNI 配置文件的网络传输带宽范围为: [100,1000]
+
+创建后，查看对应的 Multus NetworkAttachmentDefinition CR:
+
+```shell
+~# kubectl get network-attachment-definitions.k8s.cni.cncf.io -n kube-system sriov-rdma -o yaml
+apiVersion: k8s.cni.cncf.io/v1
+kind: NetworkAttachmentDefinition
+metadata:
+  name: sriov-bandwidth
+  namespace: kube-system
+  annotations:
+    k8s.v1.cni.cncf.io/resourceName: spidernet.io/sriov_netdeivce 
+  ownerReferences:
+  - apiVersion: spiderpool.spidernet.io/v2beta1
+    blockOwnerDeletion: true
+    controller: true
+    kind: SpiderMultusConfig
+    name: sriov-bandwidth
+    uid: b08ce054-1ae8-414a-b37c-7fd6988b1b8e
+spec:
+  config: '{"cniVersion":"0.3.1","name":"sriov-bandwidth","plugins":[{"vlan":100,"type":"sriov","minTxRate": 100, "maxTxRate": 1000,"ipam":{"type":"spiderpool"}},{"type":"rdma"},{"type":"coordinator"}]}'
+```
+
 ### Ifacer 使用配置
 
 Ifacer 插件可以帮助我们在创建 Pod 时，自动创建 Bond 网卡 或者 Vlan 网卡，用于承接 Pod 底层网络。更多信息参考 [Ifacer](../reference/plugin-ifacer.md)。
@@ -342,7 +481,7 @@ EOF
 
 #### 其他 CNI 配置
 
-创建其他 CNI 配置，如：SRIOV 与 Ovs，参考 [创建 SR-IOV](./install/underlay/get-started-sriov-zh_CN.md)、[创建 Ovs](./install/underlay/get-started-ovs-zh_CN.md)
+创建其他 CNI 配置，如 Ovs: 参考 [创建 Ovs](./install/underlay/get-started-ovs-zh_CN.md)
 
 ## 总结
 

diff --git a/docs/usage/spider-multus-config.md b/docs/usage/spider-multus-config.md
@@ -162,6 +162,145 @@ spec:
   config: '{"cniVersion":"0.3.1","name":"ipvlan-ens192","plugins":[{"type":"ipvlan","master":"ens192","ipam":{"type":"spiderpool"}},{"type":"coordinator"}]}'
 ```
 
+### Create Sriov Configuration
+
+Here is an example of creating Sriov SpiderMultusConfig configuration:
+
+- Basic example
+
+```bash
+cat <<EOF | kubectl apply -f -
+apiVersion: spiderpool.spidernet.io/v2beta1
+kind: SpiderMultusConfig
+metadata:
+  name: sriov-demo
+  namespace: kube-system
+spec:
+  cniType: sriov
+  enableCoordinator: true
+  sriov:
+    resourceName: spidernet.io/sriov_netdeivce
+    vlanID: 100
+EOF
+```
+
+After creation, check the corresponding Multus NetworkAttachmentDefinition CR:
+
+```shell
+~# kubectl get network-attachment-definitions.k8s.cni.cncf.io -n kube-system sriov-demo -o yaml
+apiVersion: k8s.cni.cncf.io/v1
+kind: NetworkAttachmentDefinition
+metadata:
+  name: sriov-demo
+  namespace: kube-system
+  annotations:
+    k8s.v1.cni.cncf.io/resourceName: spidernet.io/sriov_netdeivce 
+  ownerReferences:
+  - apiVersion: spiderpool.spidernet.io/v2beta1
+    blockOwnerDeletion: true
+    controller: true
+    kind: SpiderMultusConfig
+    name: sriov-demo
+    uid: b08ce054-1ae8-414a-b37c-7fd6988b1b8e
+  resourceVersion: "153002297"
+  uid: 4413e1fa-ce15-4acf-bce8-48b5028c0568
+spec:
+  config: '{"cniVersion":"0.3.1","name":"sriov-demo","plugins":[{"vlan":100,"type":"sriov","ipam":{"type":"spiderpool"}},{"type":"coordinator"}]}'
+```
+
+For more information, refer to [sriov-cni usage](./install/underlay/get-started-sriov.md)
+
+- Enable RDMA feature
+
+```shell
+cat <<EOF | kubectl apply -f -
+apiVersion: spiderpool.spidernet.io/v2beta1
+kind: SpiderMultusConfig
+metadata:
+  name: sriov-rdma
+  namespace: kube-system
+spec:
+  cniType: sriov
+  enableCoordinator: true
+  sriov:
+    enableRdma: true
+    resourceName: spidernet.io/sriov_netdeivce
+    vlanID: 100
+EOF
+```
+
+After creation, check the corresponding Multus NetworkAttachmentDefinition CR:
+
+```shell
+~# kubectl get network-attachment-definitions.k8s.cni.cncf.io -n kube-system sriov-rdma -o yaml
+apiVersion: k8s.cni.cncf.io/v1
+kind: NetworkAttachmentDefinition
+metadata:
+  name: sriov-rdma
+  namespace: kube-system
+  annotations:
+    k8s.v1.cni.cncf.io/resourceName: spidernet.io/sriov_netdeivce 
+  ownerReferences:
+  - apiVersion: spiderpool.spidernet.io/v2beta1
+    blockOwnerDeletion: true
+    controller: true
+    kind: SpiderMultusConfig
+    name: sriov-rdma
+    uid: b08ce054-1ae8-414a-b37c-7fd6988b1b8e
+  resourceVersion: "153002297"
+  uid: 4413e1fa-ce15-4acf-bce8-48b5028c0568
+spec:
+  config: '{"cniVersion":"0.3.1","name":"sriov-rdma","plugins":[{"vlan":100,"type":"sriov","ipam":{"type":"spiderpool"}},{"type":"rdma"},{"type":"coordinator"}]}'
+```
+
+For more information, refer to [Sriov-rdma usage](rdma.md)
+
+- Configure Sriov-CNI Network Bandwidth
+
+We can configure the network bandwidth of Sriov through SpiderMultusConfig:
+
+```shell
+cat <<EOF | kubectl apply -f -
+apiVersion: spiderpool.spidernet.io/v2beta1
+kind: SpiderMultusConfig
+metadata:
+  name: sriov-bandwidth
+  namespace: kube-system
+spec:
+  cniType: sriov
+  enableCoordinator: true
+  sriov:
+    resourceName: spidernet.io/sriov_netdeivce
+    vlanID: 100
+    minTxRateMbps: 100
+    MaxTxRateMbps: 1000
+EOF
+```
+
+> minTxRateMbps and MaxTxRateMbps configure the transmission bandwidth range for pods created with this configuration: [100,1000].
+
+After creation, check the corresponding Multus NetworkAttachmentDefinition CR:
+
+```shell
+~# kubectl get network-attachment-definitions.k8s.cni.cncf.io -n kube-system sriov-rdma -o yaml
+apiVersion: k8s.cni.cncf.io/v1
+kind: NetworkAttachmentDefinition
+metadata:
+  name: sriov-bandwidth
+  namespace: kube-system
+  annotations:
+    k8s.v1.cni.cncf.io/resourceName: spidernet.io/sriov_netdeivce 
+  ownerReferences:
+  - apiVersion: spiderpool.spidernet.io/v2beta1
+    blockOwnerDeletion: true
+    controller: true
+    kind: SpiderMultusConfig
+    name: sriov-bandwidth
+    uid: b08ce054-1ae8-414a-b37c-7fd6988b1b8e
+spec:
+  config: '{"cniVersion":"0.3.1","name":"sriov-bandwidth","plugins":[{"vlan":100,"type":"sriov","minTxRate": 100, "maxTxRate": 1000,"ipam":{"type":"spiderpool"}},{"type":"rdma"},{"type":"coordinator"}]}'
+```
+
 ### Ifacer Configurations
 
 The Ifacer plug-in can help us automatically create a Bond NIC or VLAN NIC when creating a pod to undertake the pod's underlying network. For more information, refer to [Ifacer](../reference/plugin-ifacer.md).

diff --git a/pkg/k8s/apis/spiderpool.spidernet.io/v2beta1/spidermultus_types.go b/pkg/k8s/apis/spiderpool.spidernet.io/v2beta1/spidermultus_types.go
@@ -100,6 +100,14 @@ type SpiderSRIOVCniConfig struct {
 	// +kubebuilder:validation:Maximum=4094
 	VlanID *int32 `json:"vlanID,omitempty"`
 
+	// +kubebuilder:validation:Optional
+	// +kubebuilder:validation:Minimum=0
+	MinTxRateMbps *int `json:"minTxRateMbps,omitempty"` // Mbps, 0 = disable rate limiting
+
+	// +kubebuilder:validation:Optional
+	// +kubebuilder:validation:Minimum=0
+	MaxTxRateMbps *int `json:"maxTxRateMbps,omitempty"` // Mbps, 0 = disable rate limiting
+
 	// +kubebuilder:default=false
 	// +kubebuilder:validation:Optional
 	EnableRdma bool `json:"enableRdma"`

diff --git a/pkg/k8s/apis/spiderpool.spidernet.io/v2beta1/zz_generated.deepcopy.go b/pkg/k8s/apis/spiderpool.spidernet.io/v2beta1/zz_generated.deepcopy.go
diff --git a/pkg/multuscniconfig/multusconfig_informer.go b/pkg/multuscniconfig/multusconfig_informer.go
@@ -557,6 +557,14 @@ func generateSriovCNIConf(disableIPAM bool, multusConfSpec spiderpoolv2beta1.Mul
 		netConf.Vlan = multusConfSpec.SriovConfig.VlanID
 	}
 
+	if multusConfSpec.SriovConfig.MaxTxRateMbps != nil {
+		netConf.MaxTxRate = multusConfSpec.SriovConfig.MaxTxRateMbps
+	}
+
+	if multusConfSpec.SriovConfig.MinTxRateMbps != nil {
+		netConf.MinTxRate = multusConfSpec.SriovConfig.MinTxRateMbps
+	}
+
 	// set default IPPools for spiderpool cni configuration
 	if multusConfSpec.SriovConfig.SpiderpoolConfigPools != nil {
 		netConf.IPAM.DefaultIPv4IPPool = multusConfSpec.SriovConfig.SpiderpoolConfigPools.IPv4IPPool

diff --git a/pkg/multuscniconfig/multusconfig_mutate.go b/pkg/multuscniconfig/multusconfig_mutate.go
@@ -94,6 +94,14 @@ func setSriovDefaultConfig(sriovConfig *spiderpoolv2beta1.SpiderSRIOVCniConfig)
 		sriovConfig.VlanID = pointer.Int32(0)
 	}
 
+	if sriovConfig.MinTxRate == nil {
+		sriovConfig.MinTxRate = pointer.Int(0)
+	}
+
+	if sriovConfig.MaxTxRate == nil {
+		sriovConfig.MaxTxRate = pointer.Int(0)
+	}
+
 	if sriovConfig.SpiderpoolConfigPools == nil {
 		sriovConfig.SpiderpoolConfigPools = &spiderpoolv2beta1.SpiderpoolPools{
 			IPv4IPPool: []string{},

diff --git a/pkg/multuscniconfig/multusconfig_validate.go b/pkg/multuscniconfig/multusconfig_validate.go
@@ -99,6 +99,12 @@ func validateCNIConfig(multusConfig *spiderpoolv2beta1.SpiderMultusConfig) *fiel
 			}
 		}
 
+		if multusConfig.Spec.SriovConfig.MinTxRateMbps != nil && multusConfig.Spec.SriovConfig.MaxTxRateMbps != nil {
+			if *multusConfig.Spec.SriovConfig.MinTxRateMbps > *multusConfig.Spec.SriovConfig.MaxTxRateMbps {
+				return field.Invalid(sriovConfigField, *multusConfig.Spec.SriovConfig.MinTxRateMbps, "minTxRateMbps must be less than maxTxRateMbps")
+			}
+		}
+
 		if multusConfig.Spec.SriovConfig.ResourceName == "" {
 			return field.Required(sriovConfigField, fmt.Sprintf("no %s specified", sriovConfigField.Key("resourceName")))
 		}