e2e:daemonset: pod phase and events

In case the DaemonSet's pods are not running for some reason,
we want to log the pod's phase and event for
better observability.

This could help us with finding the root cause
for the reason the pod didn't start.

Signed-off-by: Talor Itzhak <[email protected]>

test/e2e/performanceprofile/functests/utils/daemonset/daemonset.go

@@ -2,14 +2,18 @@ package daemonset
 
 import (
 	"context"
+	"fmt"
 	"time"
 
 	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/util/wait"
 
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
+	"github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/events"
 	testlog "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/log"
 )
 
@@ -43,6 +47,33 @@ func IsRunning(ctx context.Context, cli client.Client, namespace, name string) (
 		}
 		return false, err
 	}
-	testlog.Infof("daemonset %q %q desired %d scheduled %d ready %d", namespace, name, ds.Status.DesiredNumberScheduled, ds.Status.CurrentNumberScheduled, ds.Status.NumberReady)
-	return ds.Status.DesiredNumberScheduled > 0 && ds.Status.DesiredNumberScheduled == ds.Status.NumberReady, nil
+	if isRunning := ds.Status.DesiredNumberScheduled > 0 && ds.Status.DesiredNumberScheduled == ds.Status.NumberReady; !isRunning {
+		return false, logEventsAndPhase(ctx, cli, ds)
+	}
+	return true, nil
+}
+
+func logEventsAndPhase(ctx context.Context, cli client.Client, ds *appsv1.DaemonSet) error {
+	podList := &corev1.PodList{}
+	err := cli.List(ctx, podList, &client.ListOptions{
+		Namespace:     ds.Namespace,
+		LabelSelector: labels.SelectorFromSet(ds.Spec.Selector.MatchLabels),
+	})
+	if err != nil {
+		return fmt.Errorf("failed to list pods for DaemonSet %s; %w", client.ObjectKeyFromObject(ds).String(), err)
+	}
+	for _, pod := range podList.Items {
+		if pod.Status.Phase != corev1.PodRunning {
+			podKey := client.ObjectKeyFromObject(&pod).String()
+			testlog.Warningf("daemonset %s pod %s is not running, expected status %s got %s", ds.Name, podKey, corev1.PodRunning, pod.Status.Phase)
+			podEvents, err := events.GetEventsForObject(cli, pod.Namespace, pod.Name, string(pod.UID))
+			if err != nil {
+				return fmt.Errorf("failed to list events for Pod %s; %w", podKey, err)
+			}
+			for _, event := range podEvents.Items {
+				testlog.Warningf("-> %s %s %s", event.Action, event.Reason, event.Message)
+			}
+		}
+	}
+	return nil
 }