add ddp selector (k8snetworkplumbingwg#199)

* add ddp selector * update Dockerfile for ddp support * add DDP support docs and sample deployment files
NicolasDichtel · Feb 12, 2020 · 30e33f1 · 30e33f1
1 parent 52dffad
commit 30e33f1
Show file tree

Hide file tree

Showing 20 changed files with 829 additions and 92 deletions.
diff --git a/cmd/sriovdp/manager.go b/cmd/sriovdp/manager.go
@@ -110,7 +110,8 @@ func (rm *resourceManager) initServers() error {
 		// Create new ResourcePool
 		glog.Infof("")
 		glog.Infof("Creating new ResourcePool: %s", rc.ResourceName)
-		rPool, err := rm.rFactory.GetResourcePool(rc, rm.netDeviceList)
+		filteredDevices := rm.getFilteredDevices(rc)
+		rPool, err := rm.rFactory.GetResourcePool(rc, filteredDevices)
 		if err != nil {
 			glog.Errorf("initServers(): error creating ResourcePool with config %+v: %q", rc, err)
 			return err
@@ -292,6 +293,72 @@ func (rm *resourceManager) addToLinkWatchList(pciAddr string) {
 	}
 }
 
+// applyFilters returned a subset PciNetDevices by applying given selectors values in the following orders:
+// "vendors", "devices", "drivers", "pfNames", "ddpProfiles".
+// Each selector gets a new sub-set of devices from the result of previous one.
+func (rm *resourceManager) getFilteredDevices(rc *types.ResourceConfig) []types.PciNetDevice {
+	filteredDevice := rm.netDeviceList
+
+	rf := rm.rFactory
+	// filter by vendor list
+	if rc.Selectors.Vendors != nil && len(rc.Selectors.Vendors) > 0 {
+		if selector, err := rf.GetSelector("vendors", rc.Selectors.Vendors); err == nil {
+			filteredDevice = selector.Filter(filteredDevice)
+		}
+	}
+
+	// filter by device list
+	if rc.Selectors.Devices != nil && len(rc.Selectors.Devices) > 0 {
+		if selector, err := rf.GetSelector("devices", rc.Selectors.Devices); err == nil {
+			filteredDevice = selector.Filter(filteredDevice)
+		}
+	}
+
+	// filter by driver list
+	if rc.Selectors.Drivers != nil && len(rc.Selectors.Drivers) > 0 {
+		if selector, err := rf.GetSelector("drivers", rc.Selectors.Drivers); err == nil {
+			filteredDevice = selector.Filter(filteredDevice)
+		}
+	}
+
+	// filter by PfNames list
+	if rc.Selectors.PfNames != nil && len(rc.Selectors.PfNames) > 0 {
+		if selector, err := rf.GetSelector("pfNames", rc.Selectors.PfNames); err == nil {
+			filteredDevice = selector.Filter(filteredDevice)
+		}
+	}
+
+	// filter by linkTypes list
+	if rc.Selectors.LinkTypes != nil && len(rc.Selectors.LinkTypes) > 0 {
+		if len(rc.Selectors.LinkTypes) > 1 {
+			glog.Warningf("Link type selector should have a single value.")
+		}
+		if selector, err := rf.GetSelector("linkTypes", rc.Selectors.LinkTypes); err == nil {
+			filteredDevice = selector.Filter(filteredDevice)
+		}
+	}
+
+	// filter by DDP Profiles list
+	if rc.Selectors.DDPProfiles != nil && len(rc.Selectors.DDPProfiles) > 0 {
+		if selector, err := rf.GetSelector("ddpProfiles", rc.Selectors.DDPProfiles); err == nil {
+			filteredDevice = selector.Filter(filteredDevice)
+		}
+	}
+
+	// filter for rdma devices
+	if rc.IsRdma {
+		rdmaDevices := make([]types.PciNetDevice, 0)
+		for _, dev := range filteredDevice {
+			if dev.GetRdmaSpec().IsRdma() {
+				rdmaDevices = append(rdmaDevices, dev)
+			}
+		}
+		filteredDevice = rdmaDevices
+	}
+
+	return filteredDevice
+}
+
 type linkWatcher struct {
 	ifName string
 	// subscribers []LinkSubscriber

diff --git a/cmd/sriovdp/manager_test.go b/cmd/sriovdp/manager_test.go
@@ -9,6 +9,7 @@ import (
 	"github.com/intel/sriov-network-device-plugin/pkg/types"
 	fake "github.com/intel/sriov-network-device-plugin/pkg/types/mocks"
 	"github.com/intel/sriov-network-device-plugin/pkg/utils"
+	pluginapi "k8s.io/kubernetes/pkg/kubelet/apis/deviceplugin/v1beta1"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/ginkgo/extensions/table"
@@ -412,4 +413,60 @@ var _ = Describe("Resource manager", func() {
 			[]types.LinkWatcher{&linkWatcher{ifName: "fakenet0"}},
 		),
 	)
+
+	Describe("getFilteredDevices", func() {
+		Context("with all types of selectors used", func() {
+			var (
+				devs []types.PciNetDevice
+				c    *types.ResourceConfig
+			)
+
+			BeforeEach(func() {
+				cp = &cliParams{
+					configFile:     "/tmp/sriovdp/test_config",
+					resourcePrefix: "test_",
+				}
+				rm = newResourceManager(cp)
+
+				devs = make([]types.PciNetDevice, 4)
+				vendors := []string{"8086", "8086", "8086", "1234"}
+				codes := []string{"1111", "1111", "1234", "4321"}
+				drivers := []string{"vfio-pci", "i40evf", "igb_uio", "igb_uio"}
+				pfNames := []string{"enp2s0f2", "ens0", "eth0", "net2"}
+				linkTypes := []string{"ether", "ether", "ether", "ether"}
+				ddpProfiles := []string{"GTP", "PPPoE", "GTP", "PPPoE"}
+				for i := range devs {
+					d := &fake.PciNetDevice{}
+					d.On("GetVendor").Return(vendors[i]).
+						On("GetDeviceCode").Return(codes[i]).
+						On("GetDriver").Return(drivers[i]).
+						On("GetPFName").Return(pfNames[i]).
+						On("GetLinkType").Return(linkTypes[i]).
+						On("GetPciAddr").Return("fake").
+						On("GetAPIDevice").Return(&pluginapi.Device{}).
+						On("GetDDPProfiles").Return(ddpProfiles[i])
+					devs[i] = d
+				}
+				rm.netDeviceList = devs
+
+				c = &types.ResourceConfig{
+					ResourceName: "fake",
+					Selectors: struct {
+						Vendors     []string `json:"vendors,omitempty"`
+						Devices     []string `json:"devices,omitempty"`
+						Drivers     []string `json:"drivers,omitempty"`
+						PfNames     []string `json:"pfNames,omitempty"`
+						LinkTypes   []string `json:"linkTypes,omitempty"`
+						DDPProfiles []string `json:"ddpProfiles,omitempty"`
+					}{[]string{"8086"}, []string{"1111"}, []string{"vfio-pci"}, []string{"enp2s0f2"}, []string{"ether"}, []string{"GTP"}},
+				}
+
+			})
+			It("should return valid resource pool", func() {
+				filteredDevices := rm.getFilteredDevices(c)
+				Expect(filteredDevices).NotTo(BeNil())
+				Expect(filteredDevices).To(HaveLen(1))
+			})
+		})
+	})
 })
diff --git a/docs/README.md b/docs/README.md
@@ -4,3 +4,4 @@ This page contains supplimentary documention that users may find useful for vari
 * [Running DPDK application in Kubernetes](dpdk/)
 * [Using UDEV Rules for changing NIC's network device name](udev/)
 * [Running RDMA application in Kubernetes](rdma/)
+* [SR-IOV network device plugin with DDP](ddp/)
diff --git a/docs/ddp/README.md b/docs/ddp/README.md
@@ -0,0 +1,186 @@
+
+
+# SR-IOV network device plugin with DDP
+Dynamic Device Personalization aka DDP allows dynamic reconfiguration of the packet processing pipeline of Intel Ethernet 700 Series to meet specific use case needs on demand, adding new packet processing pipeline configuration *profiles* to a network adapter at run time, without resetting or rebooting the server.
+
+(ref: [Dynamic Device Personalization for Intel® Ethernet 700 Series](https://software.intel.com/en-us/articles/dynamic-device-personalization-for-intel-ethernet-700-series))
+
+The SR-IOV network device plugin can be used to identify currently running DDP *profiles*, allowing it to filter Virtual Functions by their DDP profile names.
+
+The Intel Ethernet 700 Series, a DDP profile can be loaded/unloaded into the NIC using i40e kernel module(v2.7.26+) and ethtool OR using DPDK i40e pollmode driver & DPDK api. In this documentation we will cover i40e Kernel driver mode.
+
+## Pre-requisites
+ * Firmware: v6.01 or newer
+ * Driver: i40e v2.7.26 or newer
+
+Refer to [Intel downloadcenter](https://downloadcenter.intel.com/) for latest firmware and drivers for Intel Ethernet 700 Series.
+
+You can use `ethtool` to get driver and firmware information of the controller.
+
+```
+# ethtool -i enp2s0f0
+driver: i40e
+version: 2.9.21
+firmware-version: 7.00 0x80004cda 1.2154.0
+expansion-rom-version:
+bus-info: 0000:02:00.0
+supports-statistics: yes
+supports-test: yes
+supports-eeprom-access: yes
+supports-register-dump: yes
+supports-priv-flags: yes
+```
+
+## 1. Install DDP profiles
+On each node, download and extract desired DDP profiles into `/lib/firmware/intel/i40e/ddp/` directory.
+
+The latest list of available packages can be found [here]( https://downloadcenter.intel.com/search?keyword=Dynamic+Device+Personalization).
+
+For example, to download and extract the GTP profile:
+
+```
+$ wget https://downloadmirror.intel.com/27587/eng/gtp.zip
+$ unzip gtp.zip
+$ mkdir -pv /lib/firmware/intel/i40e/ddp
+$ cp gtp.pkgo /lib/firmware/intel/i40e/ddp/
+```
+
+## 2. Load a DDP profile in to the NIC
+
+Use Linux `ethtool` utility to load a DDP profile into the controller.
+> Note: You can only load DDP profile into a controller using only first Physical Function(PF0).
+```
+$ ethtool -f enp2s0f0 gtp.pkgo 100
+```
+## 3. Create SR-IOV Virtual Functions
+
+Create desired number of VFs using PF interfaces of the controller.
+
+```
+$ echo 2 > /sys/class/net/enp2s0f0/device/sriov_numvfs
+
+```
+
+## 4. Verify that correct profile is loaded
+You can use another Linux utility for Intel 700 Series called `ddptool` to query current DDP profile information. This tool can be downloaded from [here]( https://downloads.sourceforge.net/project/e1000/ddptool%20stable/ddptool-1.0.0.0/ddptool-1.0.0.0.tar.gz).
+
+```
+[root@silpixa00396659 ~]# ddptool -a
+Intel(R) Dynamic Device Personalization Tool
+DDPTool version 1.0.0.0
+Copyright (C) 2019 Intel Corporation.
+
+NIC  DevId D:B:S.F      DevName         TrackId  Version      Name
+==== ===== ============ =============== ======== ============ ==============================
+001) 1572  0000:02:00.0 enp2s0f0        80000008 1.0.3.0      GTPv1-C/U IPv4/IPv6 payload
+002) 1572  0000:02:00.1 enp2s0f1        80000008 1.0.3.0      GTPv1-C/U IPv4/IPv6 payload
+003) 1572  0000:02:00.2 enp2s0f2        80000008 1.0.3.0      GTPv1-C/U IPv4/IPv6 payload
+004) 1572  0000:02:00.3 enp2s0f3        80000008 1.0.3.0      GTPv1-C/U IPv4/IPv6 payload
+005) 154C  0000:03:02.0 N/A             80000008 1.0.3.0      GTPv1-C/U IPv4/IPv6 payload
+006) 154C  0000:03:02.1 enp3s2f1        80000008 1.0.3.0      GTPv1-C/U IPv4/IPv6 payload
+007) 154C  0000:03:02.2 enp3s2f2        80000008 1.0.3.0      GTPv1-C/U IPv4/IPv6 payload
+008) 154C  0000:03:02.3 N/A             80000008 1.0.3.0      GTPv1-C/U IPv4/IPv6 payload
+```
+
+Take note of the `Name` of the profile from ddptool output. This name will be used in `"ddpProfiles"` selector in plugin's resource pool configurations.
+
+## 5. Create resource config with DDP profile selector
+
+Create ConfigMap for device plugin:
+
+```
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: sriovdp-config
+  namespace: kube-system
+data:
+  config.json: |
+    {
+        "resourceList": [{
+                "resourceName": "x700_gtp",
+                "selectors": {
+                    "vendors": ["8086"],
+                    "devices": ["154c"],
+                    "ddpProfiles": ["GTPv1-C/U IPv4/IPv6 payload"]
+                }
+            },
+            {
+                "resourceName": "x700_pppoe",
+                "selectors": {
+                    "vendors": ["8086"],
+                    "devices": ["154c"],
+                    "ddpProfiles": ["E710 PPPoE and PPPoL2TPv2"]
+                }
+            }
+        ]
+    }
+
+```
+
+```
+$ kubectl create -f configMap.yaml
+```
+
+## 6. Deploy SR-IOV network device plugin
+Once the ConfigMap for the device plugin is created/updated you can deploy the SR-IOV network device plugin as [usual](https://github.com/intel/sriov-network-device-plugin#example-deployments). When everything is good, we should see that device plugin is able to discover VFs with DDP profile names given in the resource pool selector.
+
+```
+[root@silpixa00396659 ~]# kubectl get node node1 -o json | jq ".status.allocatable"
+{
+  "cpu": "8",
+  "ephemeral-storage": "169986638772",
+  "hugepages-1Gi": "0",
+  "hugepages-2Mi": "8Gi",
+  "intel.com/x700_gtp": "2",
+  "intel.com/x700_pppoe": "0",
+  "memory": "7880620Ki",
+  "pods": "100"
+}
+
+```
+
+## 7. Create net-attach-def CRs
+This deployment is using Multus and SR-IOV CNI plugin for Pod network attachment of SR-IOV VFs. For this, we need to create a new net-attach-def CR that reference to the new resource pool.
+
+```
+$ cat crd-sriov-gtp.yaml
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+  name: sriov-net-gtp
+  annotations:
+    k8s.v1.cni.cncf.io/resourceName: intel.com/x700_gtp
+spec:
+  config: '{
+    "cniVersion": "0.3.1",
+    "type": "sriov",
+    "name": "sriov-gtp",
+    "ipam": {
+      "type": "host-local",
+      "subnet": "10.56.217.0/24",
+      "routes": [{
+        "dst": "0.0.0.0/0"
+      }],
+      "gateway": "10.56.217.1"
+    }
+}'
+```
+
+```
+$ kubectl create -f crd-sriov-gtp.yaml
+```
+
+## 8. Deploy workloads
+Once we can verify that VFs are registered by the device plugin under correct resource pool with specific DDP profile, we can request those VFs from our workload as normal.
+
+```
+$ kubectl create -f pod-gtp.yaml
+```
+
+The sample ConfigMap and Pod specs are available in this directory.
+
+## References
+* https://software.intel.com/en-us/articles/dynamic-device-personalization-for-intel-ethernet-700-series
+* https://www.intel.com/content/www/us/en/architecture-and-technology/ethernet/dynamic-device-personalization-brief.html
+* https://builders.intel.com/docs/networkbuilders/implementing-a-high-performance-bng-with-intel-universal-nfvi-packet-forwarding-platform-technology.pdf
diff --git a/docs/ddp/configMap.yaml b/docs/ddp/configMap.yaml
@@ -0,0 +1,26 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: sriovdp-config
+  namespace: kube-system
+data:
+  config.json: |
+    {
+        "resourceList": [{
+                "resourceName": "x700_gtp",
+                "selectors": {
+                    "vendors": ["8086"],
+                    "devices": ["154c"],
+                    "ddpProfiles": ["GTPv1-C/U IPv4/IPv6 payload"]
+                }
+            },
+            {
+                "resourceName": "x700_pppoe",
+                "selectors": {
+                    "vendors": ["8086"],
+                    "devices": ["154c"],
+                    "ddpProfiles": ["E710 PPPoE and PPPoL2TPv2"]
+                }
+            }
+        ]
+    }
diff --git a/docs/ddp/crd-sriov-gtp.yaml b/docs/ddp/crd-sriov-gtp.yaml
@@ -0,0 +1,20 @@
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+  name: sriov-net-gtp
+  annotations:
+    k8s.v1.cni.cncf.io/resourceName: intel.com/x700_gtp
+spec:
+  config: '{
+    "cniVersion": "0.3.1",
+    "type": "sriov",
+    "name": "sriov-gtp",
+    "ipam": {
+      "type": "host-local",
+      "subnet": "10.56.217.0/24",
+      "routes": [{
+        "dst": "0.0.0.0/0"
+      }],
+      "gateway": "10.56.217.1"
+    }
+}'