chore: Add unit test for state-sriov-dp objects rendering and sync

Signed-off-by: Soule BA <[email protected]>

pkg/state/state_sriov_dp_test.go

@@ -14,130 +14,275 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package state
+package state_test
 
 import (
 	"context"
-	"encoding/json"
-	"fmt"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+	"sigs.k8s.io/controller-runtime/pkg/log"
 
+	appsv1 "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/core/v1"
-	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/utils/ptr"
 
 	mellanoxv1alpha1 "github.com/Mellanox/network-operator/api/v1alpha1"
 	"github.com/Mellanox/network-operator/pkg/config"
-	"github.com/Mellanox/network-operator/pkg/render"
-	"github.com/Mellanox/network-operator/pkg/testing/mocks"
-	"github.com/Mellanox/network-operator/pkg/utils"
+	"github.com/Mellanox/network-operator/pkg/state"
 )
 
-func checkRenderedDpCm(obj *unstructured.Unstructured, namespace, sriovConfig string) {
-	Expect(obj.GetKind()).To(Equal("ConfigMap"))
-	Expect(obj.Object["metadata"].(map[string]interface{})["name"].(string)).To(Equal("sriovdp-config"))
-	Expect(obj.Object["metadata"].(map[string]interface{})["namespace"].(string)).To(Equal(namespace))
-	Expect(obj.Object["data"].(map[string]interface{})["config.json"].(string)).To(Equal(sriovConfig))
-}
-
-func checkRenderedDpSA(obj *unstructured.Unstructured, namespace string) {
-	Expect(obj.GetKind()).To(Equal("ServiceAccount"))
-	Expect(obj.Object["metadata"].(map[string]interface{})["namespace"].(string)).To(Equal(namespace))
-}
-
-func checkRenderedDpDs(obj *unstructured.Unstructured, imageSpec *mellanoxv1alpha1.ImageSpec,
-	nodeAffinity string) {
-	namespace := config.FromEnv().State.NetworkOperatorResourceNamespace
-	image := imageSpec.Repository + "/" + imageSpec.Image + ":" + imageSpec.Version
-	template := obj.Object["spec"].(map[string]interface{})["template"].(map[string]interface{})
-	jsonSpec, _ := obj.MarshalJSON()
-	spec := fmt.Sprintf("%v", template)
-
-	Expect(obj.GetKind()).To(Equal("DaemonSet"))
-	Expect(obj.Object["metadata"].(map[string]interface{})["namespace"].(string)).To(Equal(namespace))
-	Expect(spec).To(ContainSubstring(image))
-	Expect(string(jsonSpec)).To(ContainSubstring(nodeAffinity))
-}
-
 var _ = Describe("SR-IOV Device Plugin State tests", func() {
+	var (
+		sriovDpState state.State
+		catalog      state.InfoCatalog
+		client       client.Client
+		namespace    string
+		renderer     state.ManifestRenderer
+	)
 
-	Context("When creating NCP with SRIOV-device-plugin", func() {
-		It("Should Apply", func() {
-			client := mocks.ControllerRuntimeClient{}
-			manifestBaseDir := "../../manifests/state-sriov-device-plugin"
+	BeforeEach(func() {
+		scheme := runtime.NewScheme()
+		Expect(mellanoxv1alpha1.AddToScheme(scheme)).NotTo(HaveOccurred())
+		Expect(v1.AddToScheme(scheme)).NotTo(HaveOccurred())
+		Expect(appsv1.AddToScheme(scheme)).NotTo(HaveOccurred())
+		client = fake.NewClientBuilder().WithScheme(scheme).Build()
+		manifestDir := "../../manifests/state-sriov-device-plugin"
+		s, r, err := state.NewStateSriovDp(client, manifestDir)
+		Expect(err).NotTo(HaveOccurred())
+		sriovDpState = s
+		renderer = r
+		catalog = getTestCatalog()
+		namespace = config.FromEnv().State.NetworkOperatorResourceNamespace
+	})
 
-			files, err := utils.GetFilesWithSuffix(manifestBaseDir, render.ManifestFileSuffix...)
+	Context("When creating NCP with SRIOV-device-plugin", func() {
+		It("should create Daemonset - minimal spec", func() {
+			By("Sync")
+			cr := getMinimalNicClusterPolicyWithSriovDp("kube-sriovdp", false)
+			status, err := sriovDpState.Sync(context.Background(), cr, catalog)
 			Expect(err).NotTo(HaveOccurred())
-			renderer := render.NewRenderer(files)
-
-			stateName := "state-SRIOV-device-plugin"
-			sriovDpState := stateSriovDp{
-				stateSkel: stateSkel{
-					name:        stateName,
-					description: "SR-IOV device plugin deployed in the cluster",
-					client:      &client,
-					renderer:    renderer,
-				},
+			Expect(status).To(BeEquivalentTo(state.SyncStateNotReady))
+			By("Verify DaemonSet")
+			ds := &appsv1.DaemonSet{}
+			err = client.Get(context.Background(), types.NamespacedName{Namespace: namespace,
+				Name: "sriov-device-plugin"}, ds)
+			Expect(err).NotTo(HaveOccurred())
+			assertCommonDaemonSetFields(ds, &cr.Spec.SriovDevicePlugin.ImageSpec, cr)
+			// expect privileged mode
+			Expect(*ds.Spec.Template.Spec.Containers[0].SecurityContext.Privileged).To(BeTrue())
+			assertSriovDpPodTemplateVolumeFields(&ds.Spec.Template, false)
+		})
+		It("should create Daemonset with CDI support when specified in CR", func() {
+			By("Sync")
+			cr := getMinimalNicClusterPolicyWithSriovDp("kube-sriovdp", true)
+			status, err := sriovDpState.Sync(context.Background(), cr, catalog)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(status).To(BeEquivalentTo(state.SyncStateNotReady))
+			By("Verify DaemonSet")
+			ds := &appsv1.DaemonSet{}
+			err = client.Get(context.Background(), types.NamespacedName{Namespace: namespace,
+				Name: "sriov-device-plugin"}, ds)
+			Expect(err).NotTo(HaveOccurred())
+			assertCommonDaemonSetFields(ds, &cr.Spec.SriovDevicePlugin.ImageSpec, cr)
+			// expect privileged mode
+			Expect(*ds.Spec.Template.Spec.Containers[0].SecurityContext.Privileged).To(BeTrue())
+			assertSriovDpPodTemplateVolumeFields(&ds.Spec.Template, true)
+		})
+	})
+	Context("Verify Sync flows", func() {
+		It("should create Daemonset, update state to Ready", func() {
+			By("Sync")
+			cr := getMinimalNicClusterPolicyWithSriovDp("kube-sriovdp", false)
+			status, err := sriovDpState.Sync(context.Background(), cr, catalog)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(status).To(BeEquivalentTo(state.SyncStateNotReady))
+			By("Verify DaemonSet")
+			ds := &appsv1.DaemonSet{}
+			err = client.Get(context.Background(), types.NamespacedName{Namespace: namespace,
+				Name: "sriov-device-plugin"}, ds)
+			Expect(err).NotTo(HaveOccurred())
+			assertCommonDaemonSetFields(ds, &cr.Spec.SriovDevicePlugin.ImageSpec, cr)
+			assertSriovDpPodTemplateVolumeFields(&ds.Spec.Template, false)
+			By("Update DaemonSet Status, and re-run Sync")
+			ds.Status = appsv1.DaemonSetStatus{
+				DesiredNumberScheduled: 1,
+				NumberAvailable:        1,
+				UpdatedNumberScheduled: 1,
 			}
+			err = client.Status().Update(context.Background(), ds)
+			Expect(err).NotTo(HaveOccurred())
+			By("Verify State is ready")
+			ctx := context.Background()
+			objs, err := renderer.GetManifestObjects(ctx, cr, catalog, log.FromContext(ctx))
+			Expect(err).NotTo(HaveOccurred())
+			status, err = getKindState(ctx, client, objs, "DaemonSet")
+			Expect(err).NotTo(HaveOccurred())
+			Expect(status).To(BeEquivalentTo(state.SyncStateReady))
+		})
 
+		It("should create Daemonset and delete if Spec is nil", func() {
+			By("Sync")
+			cr := getMinimalNicClusterPolicyWithSriovDp("kube-sriovdp", false)
+			status, err := sriovDpState.Sync(context.Background(), cr, catalog)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(status).To(BeEquivalentTo(state.SyncStateNotReady))
+			By("Verify DaemonSet")
+			ds := &appsv1.DaemonSet{}
+			err = client.Get(context.Background(), types.NamespacedName{Namespace: namespace,
+				Name: "sriov-device-plugin"}, ds)
+			Expect(err).NotTo(HaveOccurred())
+			assertCommonDaemonSetFields(ds, &cr.Spec.SriovDevicePlugin.ImageSpec, cr)
+			assertSriovDpPodTemplateVolumeFields(&ds.Spec.Template, false)
+			By("Set spec to nil and Sync")
+			cr.Spec.SriovDevicePlugin = nil
+			status, err = sriovDpState.Sync(context.Background(), cr, catalog)
 			Expect(err).NotTo(HaveOccurred())
-			Expect(sriovDpState.Name()).To(Equal(stateName))
+			Expect(status).To(BeEquivalentTo(state.SyncStateNotReady))
+			By("Verify DaemonSet is deleted")
+			ds = &appsv1.DaemonSet{}
+			err = client.Get(context.Background(), types.NamespacedName{Namespace: namespace,
+				Name: "sriov-device-plugin"}, ds)
+			Expect(errors.IsNotFound(err)).To(BeTrue())
+		})
+	})
+})
 
-			cr := &mellanoxv1alpha1.NicClusterPolicy{}
-			sriovConfig := "config"
+func getMinimalNicClusterPolicyWithSriovDp(name string, useCDI bool) *mellanoxv1alpha1.NicClusterPolicy {
+	cr := getTestClusterPolicyWithBaseFields()
+	cr.Name = "nic-cluster-policy"
 
-			imageSpec := &mellanoxv1alpha1.ImageSpec{
-				Image:      "image",
-				Repository: "Repository",
-				Version:    "v0.0",
-			}
-			dpSpec := &mellanoxv1alpha1.DevicePluginSpec{
-				ImageSpecWithConfig: mellanoxv1alpha1.ImageSpecWithConfig{
-					ImageSpec: *imageSpec,
-					Config:    &sriovConfig,
+	// add an arbitrary resource, this prevent adding defaut cpu,mem limits
+	imageSpec := addContainerResources(getTestImageSpec(), name, "5", "3")
+	dpSpec := &mellanoxv1alpha1.DevicePluginSpec{
+		ImageSpecWithConfig: mellanoxv1alpha1.ImageSpecWithConfig{
+			ImageSpec: *imageSpec,
+			Config:    ptr.To("config"),
+		},
+		UseCdi: useCDI,
+	}
+	cr.Spec.SriovDevicePlugin = dpSpec
+	return cr
+}
+
+func assertSriovDpPodTemplateVolumeFields(template *v1.PodTemplateSpec, useCdi bool) {
+	src := []v1.Volume{
+		{
+			Name: "devicesock",
+			VolumeSource: v1.VolumeSource{
+				HostPath: &v1.HostPathVolumeSource{
+					Path: "/var/lib/kubelet/device-plugins",
 				},
-			}
-			cr.Spec.SriovDevicePlugin = dpSpec
-
-			//{
-			//  "requiredDuringSchedulingIgnoredDuringExecution": {
-			//	"nodeSelectorTerms": [
-			//	  {
-			//		"matchExpressions": [
-			//		  {
-			//			"key": "node-role.kubernetes.io/master",
-			//			"operator": "DoesNotExist"
-			//		  }
-			//		]
-			//	  }
-			//	]
-			//  }
-			//}
-			nodeAffinitySpec := "{\"requiredDuringSchedulingIgnoredDuringExecution\":{\"nodeSelectorTerms\":" +
-				"[{\"matchExpressions\":[{\"key\":\"node-role.kubernetes.io/master\"," +
-				"\"operator\":\"DoesNotExist\"}]}]}}"
-
-			nodeAffinity := &v1.NodeAffinity{}
-			_ = json.Unmarshal([]byte(nodeAffinitySpec), &nodeAffinity)
-
-			cr.Spec.NodeAffinity = nodeAffinity
-
-			catalog := NewInfoCatalog()
-			catalog.Add(InfoTypeNodeInfo, &dummyProvider{})
-			catalog.Add(InfoTypeStaticConfig, &dummyProvider{})
-			catalog.Add(InfoTypeClusterType, &dummyProvider{})
-
-			objs, err := sriovDpState.GetManifestObjects(context.TODO(), cr, catalog, testLogger)
+			},
+		},
+		{
+			Name: "plugins-registry",
+			VolumeSource: v1.VolumeSource{
+				HostPath: &v1.HostPathVolumeSource{
+					Path: "/var/lib/kubelet/plugins_registry",
+				},
+			},
+		},
+		{
+			Name: "log",
+			VolumeSource: v1.VolumeSource{
+				HostPath: &v1.HostPathVolumeSource{
+					Path: "/var/log",
+				},
+			},
+		},
+		{
+			Name: "device-info",
+			VolumeSource: v1.VolumeSource{
+				HostPath: &v1.HostPathVolumeSource{
+					Path: "/var/run/k8s.cni.cncf.io/devinfo/dp",
+					Type: ptr.To(v1.HostPathDirectoryOrCreate),
+				},
+			},
+		},
+		{
+			Name: "config-volume",
+			VolumeSource: v1.VolumeSource{
+				ConfigMap: &v1.ConfigMapVolumeSource{
+					LocalObjectReference: v1.LocalObjectReference{
+						Name: "sriovdp-config",
+					},
+					Items: []v1.KeyToPath{
+						{
+							Key:  "config.json",
+							Path: "config.json",
+						},
+					},
+				},
+			},
+		},
+	}
 
-			Expect(err).NotTo(HaveOccurred())
-			Expect(len(objs)).To(Equal(3))
+	if useCdi {
+		src = append(src, []v1.Volume{
+			{
+				Name: "dynamic-cdi",
+				VolumeSource: v1.VolumeSource{
+					HostPath: &v1.HostPathVolumeSource{
+						Path: "/var/run/cdi",
+						Type: ptr.To(v1.HostPathDirectoryOrCreate),
+					},
+				},
+			},
+			{
+				Name: "host-config-volume",
+				VolumeSource: v1.VolumeSource{
+					HostPath: &v1.HostPathVolumeSource{
+						Path: "/etc/pcidp",
+						Type: ptr.To(v1.HostPathDirectoryOrCreate),
+					},
+				},
+			},
+		}...)
+	}
+	Expect(template.Spec.Volumes).To(Equal(src))
 
-			namespace := config.FromEnv().State.NetworkOperatorResourceNamespace
+	vlm := []v1.VolumeMount{
+		{
+			Name:      "devicesock",
+			MountPath: "/var/lib/kubelet/device-plugins",
+			ReadOnly:  false,
+		},
+		{
+			Name:      "plugins-registry",
+			MountPath: "/var/lib/kubelet/plugins_registry",
+			ReadOnly:  false,
+		},
+		{
+			Name:      "log",
+			MountPath: "/var/log",
+		},
+		{
+			Name:      "config-volume",
+			MountPath: "/etc/pcidp",
+		},
+		{
+			Name:      "device-info",
+			MountPath: "/var/run/k8s.cni.cncf.io/devinfo/dp",
+		},
+	}
+	if useCdi {
+		vlm = append(vlm, []v1.VolumeMount{
+			{
+				Name:      "dynamic-cdi",
+				MountPath: "/var/run/cdi",
+			},
+			{
+				Name:      "host-config-volume",
+				MountPath: "/host/etc/pcidp/",
+			},
+		}...)
+	}
 
-			checkRenderedDpCm(objs[0], namespace, sriovConfig)
-			checkRenderedDpSA(objs[1], namespace)
-			checkRenderedDpDs(objs[2], imageSpec, nodeAffinitySpec)
-		})
-	})
-})
+	Expect(template.Spec.Containers[0].VolumeMounts).To(Equal(vlm))
+}