Add API for cluster type detection (Kubernetes/Openshift) (#619)

- New API for cluster type detection added
- use the API in all states

config/rbac/role.yaml

@@ -155,6 +155,7 @@ rules:
 - apiGroups:
   - config.openshift.io
   resources:
+  - clusterversions
   - proxies
   verbs:
   - get

controllers/nicclusterpolicy_controller.go

@@ -39,6 +39,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/source"
 
 	mellanoxv1alpha1 "github.com/Mellanox/network-operator/api/v1alpha1"
+	"github.com/Mellanox/network-operator/pkg/clustertype"
 	"github.com/Mellanox/network-operator/pkg/config"
 	"github.com/Mellanox/network-operator/pkg/consts"
 	"github.com/Mellanox/network-operator/pkg/nodeinfo"
@@ -48,7 +49,8 @@ import (
 // NicClusterPolicyReconciler reconciles a NicClusterPolicy object
 type NicClusterPolicyReconciler struct {
 	client.Client
-	Scheme *runtime.Scheme
+	Scheme              *runtime.Scheme
+	ClusterTypeProvider clustertype.Provider
 
 	stateManager state.Manager
 }
@@ -74,7 +76,7 @@ type NicClusterPolicyReconciler struct {
 // +kubebuilder:rbac:groups=whereabouts.cni.cncf.io,resources=overlappingrangeipreservations,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=coordination.k8s.io,resources=leases,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=batch,resources=cronjobs,verbs=get;list;watch;create;update;patch;delete
-// +kubebuilder:rbac:groups=config.openshift.io,resources=proxies,verbs=get;list;watch
+// +kubebuilder:rbac:groups=config.openshift.io,resources=proxies;clusterversions,verbs=get;list;watch
 // +kubebuilder:rbac:groups=nv-ipam.nvidia.com,resources=ippools,verbs=get;list;watch;create;
 // +kubebuilder:rbac:groups=nv-ipam.nvidia.com,resources=ippools/status,verbs=get;update;patch;
 // +kubebuilder:rbac:groups=cert-manager.io,resources=issuers;certificates,verbs=get;list;watch;create;update;patch;delete
@@ -113,9 +115,8 @@ func (r *NicClusterPolicyReconciler) Reconcile(ctx context.Context, req ctrl.Req
 
 	// Create a new State service catalog
 	sc := state.NewInfoCatalog()
-	if instance.Spec.OFEDDriver != nil || instance.Spec.RdmaSharedDevicePlugin != nil ||
-		instance.Spec.SriovDevicePlugin != nil || instance.Spec.NicFeatureDiscovery != nil ||
-		instance.Spec.NvIpam != nil {
+	sc.Add(state.InfoTypeClusterType, r.ClusterTypeProvider)
+	if instance.Spec.OFEDDriver != nil {
 		// Create node infoProvider and add to the service catalog
 		reqLogger.V(consts.LogLevelInfo).Info("Creating Node info provider")
 		nodeList := &corev1.NodeList{}

controllers/suite_test.go

@@ -26,6 +26,7 @@ import (
 	netattdefv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+	osconfigv1 "github.com/openshift/api/config/v1"
 	"k8s.io/client-go/kubernetes/scheme"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -37,6 +38,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	mellanoxcomv1alpha1 "github.com/Mellanox/network-operator/api/v1alpha1"
+	"github.com/Mellanox/network-operator/pkg/clustertype"
 	// +kubebuilder:scaffold:imports
 )
 
@@ -81,6 +83,9 @@ var _ = BeforeSuite(func() {
 	err = netattdefv1.AddToScheme(scheme.Scheme)
 	Expect(err).NotTo(HaveOccurred())
 
+	err = osconfigv1.AddToScheme(scheme.Scheme)
+	Expect(err).NotTo(HaveOccurred())
+
 	// +kubebuilder:scaffold:scheme
 
 	k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme})
@@ -111,9 +116,13 @@ var _ = BeforeSuite(func() {
 	}).SetupWithManager(k8sManager, testSetupLog)
 	Expect(err).ToNot(HaveOccurred())
 
+	clusterTypeProvider, err := clustertype.NewProvider(context.Background(), k8sClient)
+	Expect(err).NotTo(HaveOccurred())
+
 	err = (&NicClusterPolicyReconciler{
-		Client: k8sManager.GetClient(),
-		Scheme: k8sManager.GetScheme(),
+		Client:              k8sManager.GetClient(),
+		Scheme:              k8sManager.GetScheme(),
+		ClusterTypeProvider: clusterTypeProvider,
 	}).SetupWithManager(k8sManager, testSetupLog)
 	Expect(err).ToNot(HaveOccurred())
 

deployment/network-operator/templates/role.yaml

@@ -240,6 +240,7 @@ rules:
 - apiGroups:
   - config.openshift.io
   resources:
+  - clusterversions
   - proxies
   verbs:
   - get

main.go

@@ -17,6 +17,7 @@ limitations under the License.
 package main
 
 import (
+	"context"
 	"flag"
 	"os"
 
@@ -38,6 +39,7 @@ import (
 
 	mellanoxcomv1alpha1 "github.com/Mellanox/network-operator/api/v1alpha1"
 	"github.com/Mellanox/network-operator/controllers"
+	"github.com/Mellanox/network-operator/pkg/clustertype"
 	"github.com/Mellanox/network-operator/pkg/migrate"
 	// +kubebuilder:scaffold:imports
 )
@@ -56,11 +58,17 @@ func init() {
 	// +kubebuilder:scaffold:scheme
 }
 
-func setupCRDControllers(mgr ctrl.Manager) error {
+func setupCRDControllers(ctx context.Context, c client.Client, mgr ctrl.Manager) error {
 	ctrLog := setupLog.WithName("controller")
+	clusterTypeProvider, err := clustertype.NewProvider(ctx, c)
+	if err != nil {
+		setupLog.Error(err, "unable to create cluster type provider")
+		return err
+	}
 	if err := (&controllers.NicClusterPolicyReconciler{
-		Client: mgr.GetClient(),
-		Scheme: mgr.GetScheme(),
+		Client:              mgr.GetClient(),
+		Scheme:              mgr.GetScheme(),
+		ClusterTypeProvider: clusterTypeProvider, // we want to cache information about the cluster type
 	}).SetupWithManager(mgr, ctrLog.WithName("NicClusterPolicy")); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "NicClusterPolicy")
 		return err
@@ -136,7 +144,7 @@ func main() {
 		os.Exit(1)
 	}
 
-	err = setupCRDControllers(mgr)
+	err = setupCRDControllers(stopCtx, directClient, mgr)
 	if err != nil {
 		os.Exit(1)
 	}

manifests/state-ib-kubernetes/0020-role.openshift.yaml

@@ -1,4 +1,4 @@
-{{if eq .RuntimeSpec.OSName "rhcos"}}
+{{ if .RuntimeSpec.IsOpenshift }}
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:

manifests/state-ib-kubernetes/0040-role-binding.openshift.yaml

@@ -1,4 +1,4 @@
-{{if eq .RuntimeSpec.OSName "rhcos"}}
+{{ if .RuntimeSpec.IsOpenshift }}
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:

manifests/state-ipoib-cni/0010_service-account.openshift.yaml

@@ -1,4 +1,4 @@
-{{if eq .RuntimeSpec.OSName "rhcos"}}
+{{ if .RuntimeSpec.IsOpenshift }}
 apiVersion: v1
 kind: ServiceAccount
 metadata:

manifests/state-ipoib-cni/0020_role.openshift.yaml

@@ -1,4 +1,4 @@
-{{if eq .RuntimeSpec.OSName "rhcos"}}
+{{ if .RuntimeSpec.IsOpenshift }}
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:

manifests/state-ipoib-cni/0030_rolebinding.openshift.yaml

@@ -1,4 +1,4 @@
-{{if eq .RuntimeSpec.OSName "rhcos"}}
+{{ if .RuntimeSpec.IsOpenshift }}
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:

manifests/state-ipoib-cni/0050-ipoib-cni-ds.yml

@@ -39,7 +39,7 @@ spec:
         nodeAffinity:
           {{- .NodeAffinity | yaml | nindent 10 }}
       {{- end }}
-      {{- if eq .RuntimeSpec.OSName "rhcos" }}
+      {{- if .RuntimeSpec.IsOpenshift }}
       serviceAccountName: ipoib-cni
       {{- end}}
       {{- if .CrSpec.ImagePullSecrets }}
@@ -75,7 +75,7 @@ spec:
         - name: cnibin
           hostPath:
             # /opt/cni/bin directory is read-only on OCP, so we need to use another one
-            {{- if eq .RuntimeSpec.OSName "rhcos" }}
+            {{- if .RuntimeSpec.IsOpenshift }}
             path: /var/lib/cni/bin
             {{- else}}
             path: /opt/cni/bin

manifests/state-nic-feature-discovery/010_service-account.openshift.yaml

@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-{{if eq .RuntimeSpec.OSName "rhcos"}}
+{{ if .RuntimeSpec.IsOpenshift }}
 apiVersion: v1
 kind: ServiceAccount
 metadata:

manifests/state-nic-feature-discovery/020_role.openshift.yaml

@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-{{if eq .RuntimeSpec.OSName "rhcos"}}
+{{ if .RuntimeSpec.IsOpenshift }}
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:

manifests/state-nic-feature-discovery/021_rolebinding.openshift.yaml

@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-{{if eq .RuntimeSpec.OSName "rhcos"}}
+{{ if .RuntimeSpec.IsOpenshift }}
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:

manifests/state-nic-feature-discovery/030-nic-feature-discovery-ds.yaml

@@ -35,7 +35,7 @@ spec:
         name: nic-feature-discovery
     spec:
       terminationGracePeriodSeconds: 10
-      {{ if eq .RuntimeSpec.OSName "rhcos" }}
+      {{ if .RuntimeSpec.IsOpenshift }}
       serviceAccountName: nic-feature-discovery
       {{- end }}
       tolerations:

manifests/state-nv-ipam-cni/020-role.openshift.yaml

@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-{{ if eq .RuntimeSpec.OSName "rhcos" }}
+{{ if .RuntimeSpec.IsOpenshift }}
 ---
 kind: Role
 apiVersion: rbac.authorization.k8s.io/v1

manifests/state-nv-ipam-cni/030-role-binding.openshift.yaml

@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-{{ if eq .RuntimeSpec.OSName "rhcos" }}
+{{ if .RuntimeSpec.IsOpenshift }}
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding

manifests/state-nv-ipam-cni/035-certmanager.yaml

@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-{{ if and (ne .RuntimeSpec.OSName "rhcos") (.CrSpec.EnableWebhook) }}
+{{ if and (not .RuntimeSpec.IsOpenshift) (.CrSpec.EnableWebhook) }}
 ---
 apiVersion: cert-manager.io/v1
 kind: Issuer

manifests/state-nv-ipam-cni/035-webhook-service.yaml

@@ -18,7 +18,7 @@ metadata:
   name: nv-ipam-webhook-service
   namespace: {{ .RuntimeSpec.Namespace }}
   annotations:
-    {{- if eq .RuntimeSpec.OSName "rhcos" }}
+    {{- if .RuntimeSpec.IsOpenshift }}
     service.alpha.openshift.io/serving-cert-secret-name: nv-ipam-webhook-server-cert
     {{- end }}
 spec:

manifests/state-nv-ipam-cni/035-webhook.yaml

@@ -17,7 +17,7 @@ kind: ValidatingWebhookConfiguration
 metadata:
   name: nv-ipam-validating-webhook-configuration
   annotations:
-    {{- if eq .RuntimeSpec.OSName "rhcos" }}
+    {{- if .RuntimeSpec.IsOpenshift }}
     service.beta.openshift.io/inject-cabundle: "true"
     {{- else }}
     cert-manager.io/inject-ca-from: {{ .RuntimeSpec.Namespace }}/nv-ipam-serving-cert

manifests/state-nv-ipam-cni/040-nv-ipam-node.yaml

@@ -100,7 +100,7 @@ spec:
       volumes:
         - name: cnibin
           hostPath:
-            {{- if eq .RuntimeSpec.OSName "rhcos" }}
+            {{- if .RuntimeSpec.IsOpenshift }}
             path: /var/lib/cni/bin
             {{- else }}
             path: /opt/cni/bin

manifests/state-ofed-driver/0010_service-account.openshift.yaml

@@ -1,4 +1,4 @@
-{{if eq .RuntimeSpec.OSName "rhcos"}}
+{{ if .RuntimeSpec.IsOpenshift }}
 apiVersion: v1
 kind: ServiceAccount
 metadata:

manifests/state-ofed-driver/0020_role.openshift.yaml

@@ -1,4 +1,4 @@
-{{if eq .RuntimeSpec.OSName "rhcos"}}
+{{ if .RuntimeSpec.IsOpenshift }}
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:

manifests/state-ofed-driver/0030_rolebinding.openshift.yaml

@@ -1,4 +1,4 @@
-{{if eq .RuntimeSpec.OSName "rhcos"}}
+{{ if .RuntimeSpec.IsOpenshift }}
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:

manifests/state-ofed-driver/0050_ofed-driver-ds.yaml

@@ -40,7 +40,7 @@ spec:
         - key: nvidia.com/gpu
           operator: Exists
           effect: NoSchedule
-{{if eq .RuntimeSpec.OSName "rhcos"}}
+{{ if .RuntimeSpec.IsOpenshift }}
       serviceAccountName: ofed-driver
 {{end}}
       hostNetwork: true

manifests/state-rdma-device-plugin/0010_service-account.openshift.yaml

@@ -1,4 +1,4 @@
-{{if eq .RuntimeSpec.OSName "rhcos"}}
+{{ if .RuntimeSpec.IsOpenshift }}
 apiVersion: v1
 kind: ServiceAccount
 metadata:

manifests/state-rdma-device-plugin/0020_role.openshift.yaml

@@ -1,4 +1,4 @@
-{{if eq .RuntimeSpec.OSName "rhcos"}}
+{{ if .RuntimeSpec.IsOpenshift }}
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:

manifests/state-rdma-device-plugin/0030_rolebinding.openshift.yaml

@@ -1,4 +1,4 @@
-{{if eq .RuntimeSpec.OSName "rhcos"}}
+{{ if .RuntimeSpec.IsOpenshift }}
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:

manifests/state-rdma-device-plugin/0060_rdma-shared-dev-plugin-ds.yaml

@@ -27,7 +27,7 @@ spec:
     spec:
       priorityClassName: system-node-critical
       hostNetwork: true
-{{if eq .RuntimeSpec.OSName "rhcos"}}
+{{ if .RuntimeSpec.IsOpenshift }}
       serviceAccountName: rdma-shared
 {{end}}
       tolerations:

manifests/state-sriov-device-plugin/0021_role.openshift.yaml

@@ -1,4 +1,4 @@
-{{if eq .RuntimeSpec.OSName "rhcos"}}
+{{ if .RuntimeSpec.IsOpenshift }}
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:

manifests/state-sriov-device-plugin/0022_rolebinding.openshift.yaml

@@ -1,4 +1,4 @@
-{{if eq .RuntimeSpec.OSName "rhcos"}}
+{{ if .RuntimeSpec.IsOpenshift }}
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata: