diff --git a/pkg/state/common_test.go b/pkg/state/common_test.go new file mode 100644 index 000000000..a7f7aef61 --- /dev/null +++ b/pkg/state/common_test.go @@ -0,0 +1,67 @@ +/* +2024 NVIDIA CORPORATION & AFFILIATES + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package state_test + +import ( + "github.com/Mellanox/network-operator/pkg/clustertype" + "github.com/Mellanox/network-operator/pkg/nodeinfo" + "github.com/Mellanox/network-operator/pkg/state" + "github.com/Mellanox/network-operator/pkg/staticconfig" +) + +type testProvider struct { + isOpenshift bool + cniBinDir string +} + +func (tp *testProvider) GetClusterType() clustertype.Type { + if tp.isOpenshift { + return clustertype.Openshift + } + return clustertype.Kubernetes +} + +func (tp *testProvider) IsKubernetes() bool { + return !tp.isOpenshift +} + +func (tp *testProvider) IsOpenshift() bool { + return tp.isOpenshift +} + +func (tp *testProvider) GetStaticConfig() staticconfig.StaticConfig { + return staticconfig.StaticConfig{CniBinDirectory: tp.cniBinDir} +} + +func (tp *testProvider) GetNodesAttributes(...nodeinfo.Filter) []nodeinfo.NodeAttributes { + nodeAttr := make(map[nodeinfo.AttributeType]string) + nodeAttr[nodeinfo.AttrTypeCPUArch] = "amd64" + nodeAttr[nodeinfo.AttrTypeOSName] = "ubuntu" + nodeAttr[nodeinfo.AttrTypeOSVer] = "20.04" + + return []nodeinfo.NodeAttributes{{Attributes: nodeAttr}} +} + +func getTestCatalog() state.InfoCatalog { + catalog := state.NewInfoCatalog() + tp := &testProvider{isOpenshift: false, cniBinDir: ""} + catalog.Add(state.InfoTypeNodeInfo, tp) + catalog.Add(state.InfoTypeStaticConfig, tp) + catalog.Add(state.InfoTypeClusterType, tp) + + return catalog +} diff --git a/pkg/state/state_cni_plugins.go b/pkg/state/state_cni_plugins.go index 63caeb2e8..1f4436e97 100644 --- a/pkg/state/state_cni_plugins.go +++ b/pkg/state/state_cni_plugins.go @@ -146,13 +146,17 @@ func (s *stateCNIPlugins) GetManifestObjects( if staticConfig == nil { return nil, errors.New("staticConfig provider required") } + clusterInfo := catalog.GetClusterTypeProvider() + if clusterInfo == nil { + return nil, errors.New("clusterInfo provider required") + } renderData := &CNIPluginsManifestRenderData{ CrSpec: cr.Spec.SecondaryNetwork.CniPlugins, Tolerations: cr.Spec.Tolerations, NodeAffinity: cr.Spec.NodeAffinity, RuntimeSpec: &cniRuntimeSpec{ runtimeSpec: runtimeSpec{config.FromEnv().State.NetworkOperatorResourceNamespace}, - CniBinDirectory: utils.GetCniBinDirectory(staticConfig, nil), + CniBinDirectory: utils.GetCniBinDirectory(staticConfig, clusterInfo), ContainerResources: createContainerResourcesMap(cr.Spec.SecondaryNetwork.CniPlugins.ContainerResources), }, } diff --git a/pkg/state/state_cni_plugins_test.go b/pkg/state/state_cni_plugins_test.go new file mode 100644 index 000000000..8b5f7a7e5 --- /dev/null +++ b/pkg/state/state_cni_plugins_test.go @@ -0,0 +1,368 @@ +/* +2024 NVIDIA CORPORATION & AFFILIATES + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package state_test + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + appsv1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + + mellanoxv1alpha1 "github.com/Mellanox/network-operator/api/v1alpha1" + + "github.com/Mellanox/network-operator/pkg/config" + "github.com/Mellanox/network-operator/pkg/state" + + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +var _ = Describe("CNI plugins state", func() { + var cniPluginsState state.State + var catalog state.InfoCatalog + var client client.Client + var namespace string + + BeforeEach(func() { + scheme := runtime.NewScheme() + Expect(mellanoxv1alpha1.AddToScheme(scheme)).NotTo(HaveOccurred()) + Expect(appsv1.AddToScheme(scheme)).NotTo(HaveOccurred()) + client = fake.NewClientBuilder().WithScheme(scheme).Build() + manifestDir := "../../manifests/state-container-networking-plugins" + s, _, err := state.NewStateCNIPlugins(client, scheme, manifestDir) + Expect(err).NotTo(HaveOccurred()) + cniPluginsState = s + catalog = getTestCatalog() + namespace = config.FromEnv().State.NetworkOperatorResourceNamespace + }) + + Context("Verify objects rendering", func() { + It("should create Daemonset - minimal spec", func() { + By("Sync") + cr := getMinimalNicClusterPolicyWithCNIPlugins() + status, err := cniPluginsState.Sync(context.Background(), cr, catalog) + Expect(err).NotTo(HaveOccurred()) + Expect(status).To(BeEquivalentTo(state.SyncStateNotReady)) + By("Verify DaemonSet") + ds := &appsv1.DaemonSet{} + err = client.Get(context.Background(), types.NamespacedName{Namespace: namespace, Name: "cni-plugins-ds"}, ds) + Expect(err).NotTo(HaveOccurred()) + expectedDs := getExpectedMinimalCniPluginDS() + Expect(ds.Spec).To(BeEquivalentTo(expectedDs.Spec)) + }) + + It("should create Daemonset with ImagePullSecrets when specified in CR", func() { + By("Sync") + cr := getMinimalNicClusterPolicyWithCNIPlugins() + cr.Spec.SecondaryNetwork.CniPlugins.ImagePullSecrets = []string{"myimagepullsecret"} + _, err := cniPluginsState.Sync(context.Background(), cr, catalog) + Expect(err).NotTo(HaveOccurred()) + By("Verify DaemonSet") + ds := &appsv1.DaemonSet{} + err = client.Get(context.Background(), types.NamespacedName{Namespace: namespace, Name: "cni-plugins-ds"}, ds) + Expect(err).NotTo(HaveOccurred()) + expectedDs := getExpectedMinimalCniPluginDS() + By("Verify ImagePullSecret") + expectedDs.Spec.Template.Spec.ImagePullSecrets = []v1.LocalObjectReference{ + { + Name: "myimagepullsecret", + }, + } + Expect(ds.Spec).To(BeEquivalentTo(expectedDs.Spec)) + }) + + It("should create Daemonset with user specified CNI bin dir", func() { + By("Sync") + cr := getMinimalNicClusterPolicyWithCNIPlugins() + testCniBinDIr := "/opt/mydir/cni" + catalog = state.NewInfoCatalog() + catalog.Add(state.InfoTypeClusterType, &testProvider{isOpenshift: false}) + catalog.Add(state.InfoTypeStaticConfig, &testProvider{cniBinDir: testCniBinDIr}) + status, err := cniPluginsState.Sync(context.Background(), cr, catalog) + Expect(err).NotTo(HaveOccurred()) + Expect(status).To(BeEquivalentTo(state.SyncStateNotReady)) + By("Verify DaemonSet") + ds := &appsv1.DaemonSet{} + err = client.Get(context.Background(), types.NamespacedName{Namespace: namespace, Name: "cni-plugins-ds"}, ds) + Expect(err).NotTo(HaveOccurred()) + expectedDs := getExpectedMinimalCniPluginDS() + expectedDs.Spec.Template.Spec.Volumes[0].VolumeSource.HostPath.Path = testCniBinDIr + By("Verify CNI bin dir") + Expect(ds.Spec).To(BeEquivalentTo(expectedDs.Spec)) + }) + + It("should create Daemonset with Openshift CNI bin dir", func() { + By("Sync") + cr := getMinimalNicClusterPolicyWithCNIPlugins() + catalog = state.NewInfoCatalog() + catalog.Add(state.InfoTypeClusterType, &testProvider{isOpenshift: true}) + catalog.Add(state.InfoTypeStaticConfig, &testProvider{cniBinDir: ""}) + status, err := cniPluginsState.Sync(context.Background(), cr, catalog) + Expect(err).NotTo(HaveOccurred()) + Expect(status).To(BeEquivalentTo(state.SyncStateNotReady)) + By("Verify DaemonSet") + ds := &appsv1.DaemonSet{} + err = client.Get(context.Background(), types.NamespacedName{Namespace: namespace, Name: "cni-plugins-ds"}, ds) + Expect(err).NotTo(HaveOccurred()) + expectedDs := getExpectedMinimalCniPluginDS() + expectedDs.Spec.Template.Spec.Volumes[0].VolumeSource.HostPath.Path = "/var/lib/cni/bin" + By("Verify CNI bin dir") + Expect(ds.Spec).To(BeEquivalentTo(expectedDs.Spec)) + }) + + It("should render Daemonset with Resources when specified in CR", func() { + By("Sync") + cr := getMinimalNicClusterPolicyWithCNIPlugins() + cpu, _ := resource.ParseQuantity("1") + mem, _ := resource.ParseQuantity("1Gi") + cr.Spec.SecondaryNetwork.CniPlugins.ContainerResources = []mellanoxv1alpha1.ResourceRequirements{ + { + Name: "cni-plugins", + Requests: v1.ResourceList{ + v1.ResourceCPU: cpu, + v1.ResourceMemory: mem, + }, + Limits: v1.ResourceList{ + v1.ResourceCPU: cpu, + v1.ResourceMemory: mem, + }, + }, + } + _, err := cniPluginsState.Sync(context.Background(), cr, catalog) + Expect(err).NotTo(HaveOccurred()) + By("Verify DaemonSet") + ds := &appsv1.DaemonSet{} + err = client.Get(context.Background(), types.NamespacedName{Namespace: namespace, Name: "cni-plugins-ds"}, ds) + Expect(err).NotTo(HaveOccurred()) + rq := v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: cpu, + v1.ResourceMemory: mem, + }, + Limits: v1.ResourceList{ + v1.ResourceCPU: cpu, + v1.ResourceMemory: mem, + }, + } + expectedDs := getExpectedMinimalCniPluginDS() + expectedDs.Spec.Template.Spec.Containers[0].Resources = rq + By("Verify container resources") + Expect(ds.Spec).To(BeEquivalentTo(expectedDs.Spec)) + }) + + It("should render Daemonset with NodeAffinity when specified in CR", func() { + By("Sync") + cr := getMinimalNicClusterPolicyWithCNIPlugins() + nodeAffinity := &v1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: "mykey", + Operator: v1.NodeSelectorOpExists, + }, + }, + }, + }, + }, + } + cr.Spec.NodeAffinity = nodeAffinity + _, err := cniPluginsState.Sync(context.Background(), cr, catalog) + Expect(err).NotTo(HaveOccurred()) + By("Verify DaemonSet") + ds := &appsv1.DaemonSet{} + err = client.Get(context.Background(), types.NamespacedName{Namespace: namespace, Name: "cni-plugins-ds"}, ds) + Expect(err).NotTo(HaveOccurred()) + expectedDs := getExpectedMinimalCniPluginDS() + expectedDs.Spec.Template.Spec.Affinity = &v1.Affinity{NodeAffinity: nodeAffinity} + By("Verify NodeAffinity") + Expect(ds.Spec).To(BeEquivalentTo(expectedDs.Spec)) + + }) + }) + Context("Verify Sync flows", func() { + It("should create Daemonset, update state to Ready", func() { + By("Sync") + cr := getMinimalNicClusterPolicyWithCNIPlugins() + status, err := cniPluginsState.Sync(context.Background(), cr, catalog) + Expect(err).NotTo(HaveOccurred()) + Expect(status).To(BeEquivalentTo(state.SyncStateNotReady)) + By("Verify DaemonSet") + ds := &appsv1.DaemonSet{} + err = client.Get(context.Background(), types.NamespacedName{Namespace: namespace, Name: "cni-plugins-ds"}, ds) + Expect(err).NotTo(HaveOccurred()) + expectedDs := getExpectedMinimalCniPluginDS() + Expect(ds.Spec).To(BeEquivalentTo(expectedDs.Spec)) + ds.Status = appsv1.DaemonSetStatus{ + DesiredNumberScheduled: 1, + NumberAvailable: 1, + UpdatedNumberScheduled: 1, + } + By("Update DaemonSet Status, and re-run Sync") + err = client.Status().Update(context.Background(), ds) + Expect(err).NotTo(HaveOccurred()) + status, err = cniPluginsState.Sync(context.Background(), cr, catalog) + Expect(err).NotTo(HaveOccurred()) + By("Verify State is ready") + Expect(status).To(BeEquivalentTo(state.SyncStateReady)) + }) + + It("should create Daemonset and delete if Spec is nil", func() { + By("Sync") + cr := getMinimalNicClusterPolicyWithCNIPlugins() + status, err := cniPluginsState.Sync(context.Background(), cr, catalog) + Expect(err).NotTo(HaveOccurred()) + Expect(status).To(BeEquivalentTo(state.SyncStateNotReady)) + By("Verify DaemonSet") + ds := &appsv1.DaemonSet{} + err = client.Get(context.Background(), types.NamespacedName{Namespace: namespace, Name: "cni-plugins-ds"}, ds) + Expect(err).NotTo(HaveOccurred()) + expectedDs := getExpectedMinimalCniPluginDS() + Expect(ds.Spec).To(BeEquivalentTo(expectedDs.Spec)) + By("Set spec to nil and Sync") + cr.Spec.SecondaryNetwork.CniPlugins = nil + status, err = cniPluginsState.Sync(context.Background(), cr, catalog) + Expect(err).NotTo(HaveOccurred()) + Expect(status).To(BeEquivalentTo(state.SyncStateNotReady)) + By("Verify DaemonSet is deleted") + ds = &appsv1.DaemonSet{} + err = client.Get(context.Background(), types.NamespacedName{Namespace: namespace, Name: "cni-plugins-ds"}, ds) + Expect(errors.IsNotFound(err)).To(BeTrue()) + }) + + It("should fail if static config provider not set in catalog", func() { + By("Sync") + catalog := state.NewInfoCatalog() + cr := getMinimalNicClusterPolicyWithCNIPlugins() + status, err := cniPluginsState.Sync(context.Background(), cr, catalog) + Expect(err).To(HaveOccurred()) + Expect(status).To(BeEquivalentTo(state.SyncStateError)) + }) + + It("should fail if clustertype provider not set in catalog", func() { + By("Sync") + catalog := state.NewInfoCatalog() + tp := &testProvider{isOpenshift: false, cniBinDir: ""} + catalog.Add(state.InfoTypeStaticConfig, tp) + cr := getMinimalNicClusterPolicyWithCNIPlugins() + status, err := cniPluginsState.Sync(context.Background(), cr, catalog) + Expect(err).To(HaveOccurred()) + Expect(status).To(BeEquivalentTo(state.SyncStateNotReady)) + }) + }) +}) + +func getMinimalNicClusterPolicyWithCNIPlugins() *mellanoxv1alpha1.NicClusterPolicy { + cr := &mellanoxv1alpha1.NicClusterPolicy{} + cr.Name = "nic-cluster-policy" + + secondaryNetworkSpec := &mellanoxv1alpha1.SecondaryNetworkSpec{} + secondaryNetworkSpec.CniPlugins = &mellanoxv1alpha1.ImageSpec{} + secondaryNetworkSpec.CniPlugins.Image = "myimage" + secondaryNetworkSpec.CniPlugins.Repository = "myrepo" + secondaryNetworkSpec.CniPlugins.Version = "myversion" + cr.Spec.SecondaryNetwork = secondaryNetworkSpec + return cr +} + +func getExpectedMinimalCniPluginDS() *appsv1.DaemonSet { + trueVar := true + cpu, _ := resource.ParseQuantity("100m") + mem, _ := resource.ParseQuantity("50Mi") + + return &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cni-plugins-ds", + Namespace: "nvidia-network-operator", + Labels: map[string]string{ + "tier": "node", + "app": "cni-plugins", + }, + }, + Spec: appsv1.DaemonSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "name": "cni-plugins", + }, + }, + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "name": "cni-plugins", + "tier": "node", + "app": "cni-plugins", + }, + }, + Spec: v1.PodSpec{ + HostNetwork: true, + Containers: []v1.Container{ + { + Name: "cni-plugins", + Image: "myrepo/myimage:myversion", + ImagePullPolicy: v1.PullIfNotPresent, + SecurityContext: &v1.SecurityContext{ + Privileged: &trueVar, + }, + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: cpu, + v1.ResourceMemory: mem, + }, + Limits: v1.ResourceList{ + v1.ResourceCPU: cpu, + v1.ResourceMemory: mem, + }, + }, + VolumeMounts: []v1.VolumeMount{ + { + Name: "cnibin", + MountPath: "/host/opt/cni/bin", + }, + }, + }, + }, + Volumes: []v1.Volume{ + { + Name: "cnibin", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: "/opt/cni/bin", + }, + }, + }, + }, + Tolerations: []v1.Toleration{ + { + Key: "nvidia.com/gpu", + Operator: v1.TolerationOpExists, + Effect: v1.TaintEffectNoSchedule, + }, + }, + }, + }, + }, + } +} diff --git a/pkg/state/state_multus_cni.go b/pkg/state/state_multus_cni.go index d1b41faec..7db80411e 100644 --- a/pkg/state/state_multus_cni.go +++ b/pkg/state/state_multus_cni.go @@ -143,13 +143,17 @@ func (s *stateMultusCNI) GetManifestObjects( if staticConfig == nil { return nil, errors.New("staticConfig provider required") } + clusterInfo := catalog.GetClusterTypeProvider() + if clusterInfo == nil { + return nil, errors.New("clusterInfo provider required") + } renderData := &MultusManifestRenderData{ CrSpec: cr.Spec.SecondaryNetwork.Multus, Tolerations: cr.Spec.Tolerations, NodeAffinity: cr.Spec.NodeAffinity, RuntimeSpec: &cniRuntimeSpec{ runtimeSpec: runtimeSpec{config.FromEnv().State.NetworkOperatorResourceNamespace}, - CniBinDirectory: utils.GetCniBinDirectory(staticConfig, nil), + CniBinDirectory: utils.GetCniBinDirectory(staticConfig, clusterInfo), ContainerResources: createContainerResourcesMap(cr.Spec.SecondaryNetwork.Multus.ContainerResources), }, } diff --git a/pkg/state/state_multus_cni_test.go b/pkg/state/state_multus_cni_test.go index cc4ee3940..c7dd60d06 100644 --- a/pkg/state/state_multus_cni_test.go +++ b/pkg/state/state_multus_cni_test.go @@ -58,6 +58,7 @@ var _ = Describe("Multus CNI state", func() { }} catalog = NewInfoCatalog() catalog.Add(InfoTypeStaticConfig, &dummyProvider{}) + catalog.Add(InfoTypeClusterType, &dummyProvider{}) networkOperatorResourceNamespace = config.FromEnv().State.NetworkOperatorResourceNamespace }) diff --git a/pkg/state/state_whereabouts_cni.go b/pkg/state/state_whereabouts_cni.go index 64adb56a0..cebb9815f 100644 --- a/pkg/state/state_whereabouts_cni.go +++ b/pkg/state/state_whereabouts_cni.go @@ -142,13 +142,17 @@ func (s *stateWhereaboutsCNI) GetManifestObjects( if staticConfig == nil { return nil, errors.New("staticConfig provider required") } + clusterInfo := catalog.GetClusterTypeProvider() + if clusterInfo == nil { + return nil, errors.New("clusterInfo provider required") + } renderData := &WhereaboutsManifestRenderData{ CrSpec: cr.Spec.SecondaryNetwork.IpamPlugin, Tolerations: cr.Spec.Tolerations, NodeAffinity: cr.Spec.NodeAffinity, RuntimeSpec: &cniRuntimeSpec{ runtimeSpec: runtimeSpec{config.FromEnv().State.NetworkOperatorResourceNamespace}, - CniBinDirectory: utils.GetCniBinDirectory(staticConfig, nil), + CniBinDirectory: utils.GetCniBinDirectory(staticConfig, clusterInfo), ContainerResources: createContainerResourcesMap(cr.Spec.SecondaryNetwork.IpamPlugin.ContainerResources), }, }