From a46621f292da011e29163dc364b684f97d0a2f53 Mon Sep 17 00:00:00 2001 From: Soule BA Date: Mon, 22 Jul 2024 16:06:37 +0200 Subject: [PATCH] chore: Add unit test for state-sriov-dp objects rendering and sync Signed-off-by: Soule BA --- ...igmap.yml => 0010-sriov-dp-configmap.yaml} | 0 ...yml => 0020-sriov-dp-service-account.yaml} | 0 ...onset.yml => 0030-sriov-dp-daemonset.yaml} | 0 pkg/state/state_sriov_dp_test.go | 347 +++++++++++++----- 4 files changed, 246 insertions(+), 101 deletions(-) rename manifests/state-sriov-device-plugin/{0010-sriov-dp-configmap.yml => 0010-sriov-dp-configmap.yaml} (100%) rename manifests/state-sriov-device-plugin/{0020-sriov-dp-service-acocunt.yml => 0020-sriov-dp-service-account.yaml} (100%) rename manifests/state-sriov-device-plugin/{0030-sriov-dp-daemonset.yml => 0030-sriov-dp-daemonset.yaml} (100%) diff --git a/manifests/state-sriov-device-plugin/0010-sriov-dp-configmap.yml b/manifests/state-sriov-device-plugin/0010-sriov-dp-configmap.yaml similarity index 100% rename from manifests/state-sriov-device-plugin/0010-sriov-dp-configmap.yml rename to manifests/state-sriov-device-plugin/0010-sriov-dp-configmap.yaml diff --git a/manifests/state-sriov-device-plugin/0020-sriov-dp-service-acocunt.yml b/manifests/state-sriov-device-plugin/0020-sriov-dp-service-account.yaml similarity index 100% rename from manifests/state-sriov-device-plugin/0020-sriov-dp-service-acocunt.yml rename to manifests/state-sriov-device-plugin/0020-sriov-dp-service-account.yaml diff --git a/manifests/state-sriov-device-plugin/0030-sriov-dp-daemonset.yml b/manifests/state-sriov-device-plugin/0030-sriov-dp-daemonset.yaml similarity index 100% rename from manifests/state-sriov-device-plugin/0030-sriov-dp-daemonset.yml rename to manifests/state-sriov-device-plugin/0030-sriov-dp-daemonset.yaml diff --git a/pkg/state/state_sriov_dp_test.go b/pkg/state/state_sriov_dp_test.go index a0dde8e66..f985ae26b 100644 --- a/pkg/state/state_sriov_dp_test.go +++ b/pkg/state/state_sriov_dp_test.go @@ -14,130 +14,275 @@ See the License for the specific language governing permissions and limitations under the License. */ -package state +package state_test import ( "context" - "encoding/json" - "fmt" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/log" + appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" mellanoxv1alpha1 "github.com/Mellanox/network-operator/api/v1alpha1" "github.com/Mellanox/network-operator/pkg/config" - "github.com/Mellanox/network-operator/pkg/render" - "github.com/Mellanox/network-operator/pkg/testing/mocks" - "github.com/Mellanox/network-operator/pkg/utils" + "github.com/Mellanox/network-operator/pkg/state" ) -func checkRenderedDpCm(obj *unstructured.Unstructured, namespace, sriovConfig string) { - Expect(obj.GetKind()).To(Equal("ConfigMap")) - Expect(obj.Object["metadata"].(map[string]interface{})["name"].(string)).To(Equal("sriovdp-config")) - Expect(obj.Object["metadata"].(map[string]interface{})["namespace"].(string)).To(Equal(namespace)) - Expect(obj.Object["data"].(map[string]interface{})["config.json"].(string)).To(Equal(sriovConfig)) -} - -func checkRenderedDpSA(obj *unstructured.Unstructured, namespace string) { - Expect(obj.GetKind()).To(Equal("ServiceAccount")) - Expect(obj.Object["metadata"].(map[string]interface{})["namespace"].(string)).To(Equal(namespace)) -} - -func checkRenderedDpDs(obj *unstructured.Unstructured, imageSpec *mellanoxv1alpha1.ImageSpec, - nodeAffinity string) { - namespace := config.FromEnv().State.NetworkOperatorResourceNamespace - image := imageSpec.Repository + "/" + imageSpec.Image + ":" + imageSpec.Version - template := obj.Object["spec"].(map[string]interface{})["template"].(map[string]interface{}) - jsonSpec, _ := obj.MarshalJSON() - spec := fmt.Sprintf("%v", template) - - Expect(obj.GetKind()).To(Equal("DaemonSet")) - Expect(obj.Object["metadata"].(map[string]interface{})["namespace"].(string)).To(Equal(namespace)) - Expect(spec).To(ContainSubstring(image)) - Expect(string(jsonSpec)).To(ContainSubstring(nodeAffinity)) -} - var _ = Describe("SR-IOV Device Plugin State tests", func() { + var ( + sriovDpState state.State + catalog state.InfoCatalog + client client.Client + namespace string + renderer state.ManifestRenderer + ) - Context("When creating NCP with SRIOV-device-plugin", func() { - It("Should Apply", func() { - client := mocks.ControllerRuntimeClient{} - manifestBaseDir := "../../manifests/state-sriov-device-plugin" + BeforeEach(func() { + scheme := runtime.NewScheme() + Expect(mellanoxv1alpha1.AddToScheme(scheme)).NotTo(HaveOccurred()) + Expect(v1.AddToScheme(scheme)).NotTo(HaveOccurred()) + Expect(appsv1.AddToScheme(scheme)).NotTo(HaveOccurred()) + client = fake.NewClientBuilder().WithScheme(scheme).Build() + manifestDir := "../../manifests/state-sriov-device-plugin" + s, r, err := state.NewStateSriovDp(client, manifestDir) + Expect(err).NotTo(HaveOccurred()) + sriovDpState = s + renderer = r + catalog = getTestCatalog() + namespace = config.FromEnv().State.NetworkOperatorResourceNamespace + }) - files, err := utils.GetFilesWithSuffix(manifestBaseDir, render.ManifestFileSuffix...) + Context("When creating NCP with SRIOV-device-plugin", func() { + It("should create Daemonset - minimal spec", func() { + By("Sync") + cr := getMinimalNicClusterPolicyWithSriovDp("kube-sriovdp", false) + status, err := sriovDpState.Sync(context.Background(), cr, catalog) Expect(err).NotTo(HaveOccurred()) - renderer := render.NewRenderer(files) - - stateName := "state-SRIOV-device-plugin" - sriovDpState := stateSriovDp{ - stateSkel: stateSkel{ - name: stateName, - description: "SR-IOV device plugin deployed in the cluster", - client: &client, - renderer: renderer, - }, + Expect(status).To(BeEquivalentTo(state.SyncStateNotReady)) + By("Verify DaemonSet") + ds := &appsv1.DaemonSet{} + err = client.Get(context.Background(), types.NamespacedName{Namespace: namespace, + Name: "sriov-device-plugin"}, ds) + Expect(err).NotTo(HaveOccurred()) + assertCommonDaemonSetFields(ds, &cr.Spec.SriovDevicePlugin.ImageSpec, cr) + // expect privileged mode + Expect(*ds.Spec.Template.Spec.Containers[0].SecurityContext.Privileged).To(BeTrue()) + assertSriovDpPodTemplateVolumeFields(&ds.Spec.Template, false) + }) + It("should create Daemonset with CDI support when specified in CR", func() { + By("Sync") + cr := getMinimalNicClusterPolicyWithSriovDp("kube-sriovdp", true) + status, err := sriovDpState.Sync(context.Background(), cr, catalog) + Expect(err).NotTo(HaveOccurred()) + Expect(status).To(BeEquivalentTo(state.SyncStateNotReady)) + By("Verify DaemonSet") + ds := &appsv1.DaemonSet{} + err = client.Get(context.Background(), types.NamespacedName{Namespace: namespace, + Name: "sriov-device-plugin"}, ds) + Expect(err).NotTo(HaveOccurred()) + assertCommonDaemonSetFields(ds, &cr.Spec.SriovDevicePlugin.ImageSpec, cr) + // expect privileged mode + Expect(*ds.Spec.Template.Spec.Containers[0].SecurityContext.Privileged).To(BeTrue()) + assertSriovDpPodTemplateVolumeFields(&ds.Spec.Template, true) + }) + }) + Context("Verify Sync flows", func() { + It("should create Daemonset, update state to Ready", func() { + By("Sync") + cr := getMinimalNicClusterPolicyWithSriovDp("kube-sriovdp", false) + status, err := sriovDpState.Sync(context.Background(), cr, catalog) + Expect(err).NotTo(HaveOccurred()) + Expect(status).To(BeEquivalentTo(state.SyncStateNotReady)) + By("Verify DaemonSet") + ds := &appsv1.DaemonSet{} + err = client.Get(context.Background(), types.NamespacedName{Namespace: namespace, + Name: "sriov-device-plugin"}, ds) + Expect(err).NotTo(HaveOccurred()) + assertCommonDaemonSetFields(ds, &cr.Spec.SriovDevicePlugin.ImageSpec, cr) + assertSriovDpPodTemplateVolumeFields(&ds.Spec.Template, false) + By("Update DaemonSet Status, and re-run Sync") + ds.Status = appsv1.DaemonSetStatus{ + DesiredNumberScheduled: 1, + NumberAvailable: 1, + UpdatedNumberScheduled: 1, } + err = client.Status().Update(context.Background(), ds) + Expect(err).NotTo(HaveOccurred()) + By("Verify State is ready") + ctx := context.Background() + objs, err := renderer.GetManifestObjects(ctx, cr, catalog, log.FromContext(ctx)) + Expect(err).NotTo(HaveOccurred()) + status, err = getKindState(ctx, client, objs, "DaemonSet") + Expect(err).NotTo(HaveOccurred()) + Expect(status).To(BeEquivalentTo(state.SyncStateReady)) + }) + It("should create Daemonset and delete if Spec is nil", func() { + By("Sync") + cr := getMinimalNicClusterPolicyWithSriovDp("kube-sriovdp", false) + status, err := sriovDpState.Sync(context.Background(), cr, catalog) + Expect(err).NotTo(HaveOccurred()) + Expect(status).To(BeEquivalentTo(state.SyncStateNotReady)) + By("Verify DaemonSet") + ds := &appsv1.DaemonSet{} + err = client.Get(context.Background(), types.NamespacedName{Namespace: namespace, + Name: "sriov-device-plugin"}, ds) + Expect(err).NotTo(HaveOccurred()) + assertCommonDaemonSetFields(ds, &cr.Spec.SriovDevicePlugin.ImageSpec, cr) + assertSriovDpPodTemplateVolumeFields(&ds.Spec.Template, false) + By("Set spec to nil and Sync") + cr.Spec.SriovDevicePlugin = nil + status, err = sriovDpState.Sync(context.Background(), cr, catalog) Expect(err).NotTo(HaveOccurred()) - Expect(sriovDpState.Name()).To(Equal(stateName)) + Expect(status).To(BeEquivalentTo(state.SyncStateNotReady)) + By("Verify DaemonSet is deleted") + ds = &appsv1.DaemonSet{} + err = client.Get(context.Background(), types.NamespacedName{Namespace: namespace, + Name: "sriov-device-plugin"}, ds) + Expect(errors.IsNotFound(err)).To(BeTrue()) + }) + }) +}) - cr := &mellanoxv1alpha1.NicClusterPolicy{} - sriovConfig := "config" +func getMinimalNicClusterPolicyWithSriovDp(name string, useCDI bool) *mellanoxv1alpha1.NicClusterPolicy { + cr := getTestClusterPolicyWithBaseFields() + cr.Name = "nic-cluster-policy" - imageSpec := &mellanoxv1alpha1.ImageSpec{ - Image: "image", - Repository: "Repository", - Version: "v0.0", - } - dpSpec := &mellanoxv1alpha1.DevicePluginSpec{ - ImageSpecWithConfig: mellanoxv1alpha1.ImageSpecWithConfig{ - ImageSpec: *imageSpec, - Config: &sriovConfig, + // add an arbitrary resource, this prevent adding defaut cpu,mem limits + imageSpec := addContainerResources(getTestImageSpec(), name, "5", "3") + dpSpec := &mellanoxv1alpha1.DevicePluginSpec{ + ImageSpecWithConfig: mellanoxv1alpha1.ImageSpecWithConfig{ + ImageSpec: *imageSpec, + Config: ptr.To("config"), + }, + UseCdi: useCDI, + } + cr.Spec.SriovDevicePlugin = dpSpec + return cr +} + +func assertSriovDpPodTemplateVolumeFields(template *v1.PodTemplateSpec, useCdi bool) { + src := []v1.Volume{ + { + Name: "devicesock", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: "/var/lib/kubelet/device-plugins", }, - } - cr.Spec.SriovDevicePlugin = dpSpec - - //{ - // "requiredDuringSchedulingIgnoredDuringExecution": { - // "nodeSelectorTerms": [ - // { - // "matchExpressions": [ - // { - // "key": "node-role.kubernetes.io/master", - // "operator": "DoesNotExist" - // } - // ] - // } - // ] - // } - //} - nodeAffinitySpec := "{\"requiredDuringSchedulingIgnoredDuringExecution\":{\"nodeSelectorTerms\":" + - "[{\"matchExpressions\":[{\"key\":\"node-role.kubernetes.io/master\"," + - "\"operator\":\"DoesNotExist\"}]}]}}" - - nodeAffinity := &v1.NodeAffinity{} - _ = json.Unmarshal([]byte(nodeAffinitySpec), &nodeAffinity) - - cr.Spec.NodeAffinity = nodeAffinity - - catalog := NewInfoCatalog() - catalog.Add(InfoTypeNodeInfo, &dummyProvider{}) - catalog.Add(InfoTypeStaticConfig, &dummyProvider{}) - catalog.Add(InfoTypeClusterType, &dummyProvider{}) - - objs, err := sriovDpState.GetManifestObjects(context.TODO(), cr, catalog, testLogger) + }, + }, + { + Name: "plugins-registry", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: "/var/lib/kubelet/plugins_registry", + }, + }, + }, + { + Name: "log", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: "/var/log", + }, + }, + }, + { + Name: "device-info", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: "/var/run/k8s.cni.cncf.io/devinfo/dp", + Type: ptr.To(v1.HostPathDirectoryOrCreate), + }, + }, + }, + { + Name: "config-volume", + VolumeSource: v1.VolumeSource{ + ConfigMap: &v1.ConfigMapVolumeSource{ + LocalObjectReference: v1.LocalObjectReference{ + Name: "sriovdp-config", + }, + Items: []v1.KeyToPath{ + { + Key: "config.json", + Path: "config.json", + }, + }, + }, + }, + }, + } - Expect(err).NotTo(HaveOccurred()) - Expect(len(objs)).To(Equal(3)) + if useCdi { + src = append(src, []v1.Volume{ + { + Name: "dynamic-cdi", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: "/var/run/cdi", + Type: ptr.To(v1.HostPathDirectoryOrCreate), + }, + }, + }, + { + Name: "host-config-volume", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: "/etc/pcidp", + Type: ptr.To(v1.HostPathDirectoryOrCreate), + }, + }, + }, + }...) + } + Expect(template.Spec.Volumes).To(Equal(src)) - namespace := config.FromEnv().State.NetworkOperatorResourceNamespace + vlm := []v1.VolumeMount{ + { + Name: "devicesock", + MountPath: "/var/lib/kubelet/device-plugins", + ReadOnly: false, + }, + { + Name: "plugins-registry", + MountPath: "/var/lib/kubelet/plugins_registry", + ReadOnly: false, + }, + { + Name: "log", + MountPath: "/var/log", + }, + { + Name: "config-volume", + MountPath: "/etc/pcidp", + }, + { + Name: "device-info", + MountPath: "/var/run/k8s.cni.cncf.io/devinfo/dp", + }, + } + if useCdi { + vlm = append(vlm, []v1.VolumeMount{ + { + Name: "dynamic-cdi", + MountPath: "/var/run/cdi", + }, + { + Name: "host-config-volume", + MountPath: "/host/etc/pcidp/", + }, + }...) + } - checkRenderedDpCm(objs[0], namespace, sriovConfig) - checkRenderedDpSA(objs[1], namespace) - checkRenderedDpDs(objs[2], imageSpec, nodeAffinitySpec) - }) - }) -}) + Expect(template.Spec.Containers[0].VolumeMounts).To(Equal(vlm)) +}