Merge pull request #30 from almaslennikov/disable-roce-for-ib

fix: disable RoCE-specific settings for IB devices

README.md

@@ -116,6 +116,7 @@ spec:
   * Configure pfc (Priority Flow Control) for priority 3 and set trust to dscp on each PF
     * Non-persistent (need to be applied after each boot)
     * Users can override values via `trust` and `pfc` parameters
+  * Can only be enabled with `linkType=Ethernet`
 * `gpuDirectOptimized`: performs gpu direct optimizations. ATM only optimizations for Baremetal environment are supported. If enabled perform the following:
   * Set nvconfig `ATS_ENABLED=0`
   * Can only be enabled when `pciPerformanceOptimized` is enabled

pkg/host/configvalidation.go

@@ -135,6 +135,13 @@ func (v *configValidationImpl) ConstructNvParamMapFromTemplate(
 	}
 
 	if template.RoceOptimized != nil && template.RoceOptimized.Enabled {
+		if template.LinkType == consts.Infiniband {
+			err := types.IncorrectSpecError(
+				"RoceOptimized settings can only be used with link type Ethernet")
+			log.Log.Error(err, "incorrect spec", "device", device.Name)
+			return desiredParameters, err
+		}
+
 		desiredParameters[consts.RoceCcPrioMaskP1Param] = "255"
 		desiredParameters[consts.CnpDscpP1Param] = "4"
 		desiredParameters[consts.Cnp802pPrioP1Param] = "6"
@@ -280,8 +287,6 @@ func (v *configValidationImpl) RuntimeConfigApplied(device *v1alpha1.NicDevice)
 // returns string - qos pfc settings
 func (v *configValidationImpl) CalculateDesiredRuntimeConfig(device *v1alpha1.NicDevice) (int, string, string) {
 	maxReadRequestSize := 0
-	trust := "pcp"
-	pfc := "0,0,0,0,0,0,0,0"
 
 	template := device.Spec.Configuration.Template
 
@@ -293,6 +298,14 @@ func (v *configValidationImpl) CalculateDesiredRuntimeConfig(device *v1alpha1.Ni
 		}
 	}
 
+	// QoS settings are not available for IB devices
+	if template.LinkType == consts.Infiniband {
+		return maxReadRequestSize, "", ""
+	}
+
+	trust := "pcp"
+	pfc := "0,0,0,0,0,0,0,0"
+
 	if template.RoceOptimized != nil && template.RoceOptimized.Enabled {
 		trust = "dscp"
 		pfc = "0,0,0,1,0,0,0,0"

pkg/host/configvalidation_test.go

@@ -259,7 +259,7 @@ var _ = Describe("ConfigValidationImpl", func() {
 			defaultValues := map[string]string{}
 
 			_, err := validator.ConstructNvParamMapFromTemplate(device, defaultValues)
-			Expect(err).To(HaveOccurred())
+			Expect(err).To(MatchError("incorrect spec: GpuDirectOptimized should only be enabled together with PciPerformanceOptimized"))
 		})
 		It("should ignore raw config for the second port if device is single port", func() {
 			mockHostUtils.On("GetPCILinkSpeed", mock.Anything).Return(16, nil)
@@ -441,6 +441,33 @@ var _ = Describe("ConfigValidationImpl", func() {
 			_, err := validator.ConstructNvParamMapFromTemplate(device, defaultValues)
 			Expect(err).NotTo(HaveOccurred())
 		})
+		It("should return an error when RoceOptimized is enabled with linkType Infiniband", func() {
+			mockHostUtils.On("GetPCILinkSpeed", mock.Anything).Return(16, nil)
+
+			device := &v1alpha1.NicDevice{
+				Spec: v1alpha1.NicDeviceSpec{
+					Configuration: &v1alpha1.NicDeviceConfigurationSpec{
+						Template: &v1alpha1.ConfigurationTemplateSpec{
+							NumVfs:   0,
+							LinkType: consts.Infiniband,
+							RoceOptimized: &v1alpha1.RoceOptimizedSpec{
+								Enabled: true,
+							},
+						},
+					},
+				},
+				Status: v1alpha1.NicDeviceStatus{
+					Ports: []v1alpha1.NicDevicePortSpec{
+						{PCI: "0000:03:00.0"},
+					},
+				},
+			}
+
+			defaultValues := map[string]string{}
+
+			_, err := validator.ConstructNvParamMapFromTemplate(device, defaultValues)
+			Expect(err).To(MatchError("incorrect spec: RoceOptimized settings can only be used with link type Ethernet"))
+		})
 	})
 
 	Describe("ValidateResetToDefault", func() {
@@ -621,6 +648,34 @@ var _ = Describe("ConfigValidationImpl", func() {
 			Expect(trust).To(Equal("customTrust"))
 			Expect(pfc).To(Equal("1,1,1,1,1,1,1,1"))
 		})
+
+		It("should not calculate desired QoS settings for an IB configuration", func() {
+			device := &v1alpha1.NicDevice{
+				Spec: v1alpha1.NicDeviceSpec{
+					Configuration: &v1alpha1.NicDeviceConfigurationSpec{
+						Template: &v1alpha1.ConfigurationTemplateSpec{
+							LinkType: consts.Infiniband,
+							PciPerformanceOptimized: &v1alpha1.PciPerformanceOptimizedSpec{
+								Enabled:        true,
+								MaxReadRequest: 256,
+							},
+							RoceOptimized: &v1alpha1.RoceOptimizedSpec{
+								Enabled: true,
+								Qos: &v1alpha1.QosSpec{
+									Trust: "customTrust",
+									PFC:   "1,1,1,1,1,1,1,1",
+								},
+							},
+						},
+					},
+				},
+			}
+
+			maxReadRequestSize, trust, pfc := validator.CalculateDesiredRuntimeConfig(device)
+			Expect(maxReadRequestSize).To(Equal(256))
+			Expect(trust).To(BeEmpty())
+			Expect(pfc).To(BeEmpty())
+		})
 	})
 
 	Describe("RuntimeConfigApplied", func() {