WIP

Signed-off-by: Sebastian Sch <[email protected]>

Dockerfile

@@ -5,6 +5,7 @@ RUN make _build-manager BIN_PATH=build/_output/cmd
 RUN make _build-sriov-network-operator-config-cleanup BIN_PATH=build/_output/cmd
 
 FROM quay.io/centos/centos:stream9
+RUN yum -y install delve procps-ng
 COPY --from=builder /go/src/github.com/k8snetworkplumbingwg/sriov-network-operator/build/_output/cmd/manager /usr/bin/sriov-network-operator
 COPY --from=builder /go/src/github.com/k8snetworkplumbingwg/sriov-network-operator/build/_output/cmd/sriov-network-operator-config-cleanup /usr/bin/sriov-network-operator-config-cleanup
 COPY bindata /bindata

Dockerfile.sriov-network-config-daemon

@@ -7,7 +7,7 @@ FROM quay.io/centos/centos:stream9
 ARG MSTFLINT=mstflint
 # We have to ensure that pciutils is installed. This package is needed for mstfwreset to succeed.
 # xref pkg/vendors/mellanox/mellanox.go#L150
-RUN ARCH_DEP_PKGS=$(if [ "$(uname -m)" != "s390x" ]; then echo -n ${MSTFLINT} ; fi) && yum -y install hwdata pciutils $ARCH_DEP_PKGS && yum clean all
+RUN ARCH_DEP_PKGS=$(if [ "$(uname -m)" != "s390x" ]; then echo -n ${MSTFLINT} ; fi) && yum -y install delve procps-ng hwdata pciutils $ARCH_DEP_PKGS && yum clean all
 LABEL io.k8s.display-name="sriov-network-config-daemon" \
       io.k8s.description="This is a daemon that manage and config sriov network devices in Kubernetes cluster"
 COPY --from=builder /go/src/github.com/k8snetworkplumbingwg/sriov-network-operator/build/_output/cmd/sriov-network-config-daemon /usr/bin/

api/v1/helper.go

@@ -8,6 +8,8 @@ import (
 	"path/filepath"
 	"reflect"
 	"regexp"
+	"sigs.k8s.io/controller-runtime/pkg/event"
+	"sigs.k8s.io/controller-runtime/pkg/predicate"
 	"slices"
 	"sort"
 	"strconv"
@@ -1005,3 +1007,66 @@ func GenerateBridgeName(iface *InterfaceExt) string {
 func NeedToUpdateBridges(bridgeSpec, bridgeStatus *Bridges) bool {
 	return !reflect.DeepEqual(bridgeSpec, bridgeStatus)
 }
+
+type DrainAnnotationPredicate struct {
+	predicate.Funcs
+}
+
+func (DrainAnnotationPredicate) Create(e event.CreateEvent) bool {
+	if e.Object == nil {
+		return false
+	}
+
+	if _, hasAnno := e.Object.GetAnnotations()[consts.NodeDrainAnnotation]; hasAnno {
+		return true
+	}
+	return false
+}
+
+func (DrainAnnotationPredicate) Update(e event.UpdateEvent) bool {
+	if e.ObjectOld == nil {
+		return false
+	}
+	if e.ObjectNew == nil {
+		return false
+	}
+
+	oldAnno, hasOldAnno := e.ObjectOld.GetAnnotations()[consts.NodeDrainAnnotation]
+	newAnno, hasNewAnno := e.ObjectNew.GetAnnotations()[consts.NodeDrainAnnotation]
+
+	if !hasOldAnno && hasNewAnno {
+		return true
+	}
+
+	return oldAnno != newAnno
+}
+
+type DrainStateAnnotationPredicate struct {
+	predicate.Funcs
+}
+
+func (DrainStateAnnotationPredicate) Create(e event.CreateEvent) bool {
+	if e.Object == nil {
+		return false
+	}
+
+	return true
+}
+
+func (DrainStateAnnotationPredicate) Update(e event.UpdateEvent) bool {
+	if e.ObjectOld == nil {
+		return false
+	}
+	if e.ObjectNew == nil {
+		return false
+	}
+
+	oldAnno, hasOldAnno := e.ObjectOld.GetLabels()[consts.NodeStateDrainAnnotationCurrent]
+	newAnno, hasNewAnno := e.ObjectNew.GetLabels()[consts.NodeStateDrainAnnotationCurrent]
+
+	if !hasOldAnno || !hasNewAnno {
+		return true
+	}
+
+	return oldAnno != newAnno
+}

bindata/manifests/daemon/daemonset.yaml

@@ -130,12 +130,24 @@ spec:
       containers:
       - name: sriov-network-config-daemon
         image: {{.Image}}
-        command:
-          - sriov-network-config-daemon
+        imagePullPolicy: Always
         securityContext:
           privileged: true
+        command:
+          - sriov-network-config-daemon
         args:
           - "start"
+#        command:
+#          - dlv
+#          - --listen=:2345
+#          - --headless=true
+#          - --api-version=2
+#          - --accept-multiclient
+#          - --log
+#          - exec
+#          - /usr/bin/sriov-network-config-daemon
+#          - --
+#          - start
         {{- if .UsedSystemdMode}}
           - --use-systemd-service
         {{- end }}

bindata/manifests/operator-webhook/server.yaml

@@ -64,6 +64,7 @@ spec:
       containers:
       - name: webhook-server
         image: {{.SriovNetworkWebhookImage}}
+        imagePullPolicy: Always
         command:
         - webhook
         args:

bindata/manifests/plugins/sriov-device-plugin.yaml

@@ -42,6 +42,7 @@ spec:
       containers:
       - name: sriov-device-plugin
         image: {{.SRIOVDevicePluginImage}}
+        imagePullPolicy: Always
         args:
         - --log-level=10
         - --resource-prefix={{.ResourcePrefix}}

bindata/manifests/sriov-config-service/kubernetes/sriov-config-post-network-service.yaml

@@ -6,7 +6,7 @@ contents: |
 
   [Service]
   Type=oneshot
-  ExecStart=/var/lib/sriov/sriov-network-config-daemon -v 2 --zap-log-level 2 service --phase post
+  ExecStart=/var/lib/sriov/sriov-network-config-daemon service --phase post
   StandardOutput=journal+console
 
   [Install]

bindata/manifests/sriov-config-service/kubernetes/sriov-config-service.yaml

@@ -7,7 +7,7 @@ contents: |
 
   [Service]
   Type=oneshot
-  ExecStart=/var/lib/sriov/sriov-network-config-daemon -v 2 --zap-log-level 2 service --phase pre
+  ExecStart=/var/lib/sriov/sriov-network-config-daemon service --phase pre
   StandardOutput=journal+console
 
   [Install]

bindata/manifests/sriov-config-service/openshift/sriov-config-service.yaml

@@ -21,7 +21,7 @@ spec:
 
             [Service]
             Type=oneshot
-            ExecStart=/var/lib/sriov/sriov-network-config-daemon service -v {{ .LogLevel }} --zap-log-level {{ .LogLevel }} --phase pre
+            ExecStart=/var/lib/sriov/sriov-network-config-daemon service --phase pre
             StandardOutput=journal+console
 
             [Install]
@@ -38,7 +38,7 @@ spec:
 
             [Service]
             Type=oneshot
-            ExecStart=/var/lib/sriov/sriov-network-config-daemon service -v {{ .LogLevel }} --zap-log-level {{ .LogLevel }} --phase post
+            ExecStart=/var/lib/sriov/sriov-network-config-daemon service --phase post
             StandardOutput=journal+console
 
             [Install]

bindata/manifests/webhook/server.yaml

@@ -67,6 +67,7 @@ spec:
       containers:
       - name: webhook-server
         image: {{.NetworkResourcesInjectorImage}}
+        imagePullPolicy: Always
         command:
         - webhook
         args:

cmd/sriov-network-config-daemon/service.go

@@ -85,6 +85,7 @@ func runServiceCmd(cmd *cobra.Command, args []string) error {
 	}
 	// init logger
 	snolog.InitLog()
+	snolog.SetLogLevel(2)
 	setupLog := log.Log.WithName("sriov-config-service").WithValues("phase", phaseArg)
 
 	setupLog.V(0).Info("Starting sriov-config-service", "version", version.Version)
@@ -163,12 +164,24 @@ func phasePre(setupLog logr.Logger, conf *systemd.SriovConfig, hostHelpers helpe
 	hostHelpers.TryEnableTun()
 	hostHelpers.TryEnableVhostNet()
 
-	return callPlugin(setupLog, PhasePre, conf, hostHelpers)
+	// Add retry here as some time the kernel is not fully loaded, and we see issues like
+	// unbindDriver(): failed to unbind driver "error": "open /sys/bus/pci/drivers/igbvf/unbind: permission denied"
+	i := 0
+	for i < 5 {
+		err = callPlugin(setupLog, PhasePre, conf, hostHelpers)
+		if err == nil {
+			break
+		}
+		i++
+		time.Sleep(5 * time.Second)
+	}
+
+	return err
 }
 
 func phasePost(setupLog logr.Logger, conf *systemd.SriovConfig, hostHelpers helper.HostHelpersInterface) error {
 	setupLog.V(0).Info("check result of the Pre phase")
-	prePhaseResult, err := systemd.ReadSriovResult()
+	prePhaseResult, _, err := systemd.ReadSriovResult()
 	if err != nil {
 		return fmt.Errorf("failed to read result of the pre phase: %v", err)
 	}