system-test: Add stability test for ran du

This commit adds two new test cases for stability in RAN env:
- Stability without workload runnning
- Stability with workload runing

Both tests collect a series of metrics and verify changes on them
at the end of the test duration

Signed-off-by: Rodrigo Lopez <[email protected]>

tests/system-tests/internal/ptp/ptp.go

@@ -0,0 +1,125 @@
+package ptp
+
+import (
+	"fmt"
+	"regexp"
+	"strings"
+	"time"
+
+	"github.com/openshift-kni/eco-goinfra/pkg/clients"
+	"github.com/openshift-kni/eco-goinfra/pkg/pod"
+)
+
+const (
+	machineConfigNamespace = "openshift-machine-config-operator"
+	machineConfigDaemonPod = "machine-config-daemon"
+	ptpNamespace           = "openshift-ptp"
+	ptpLinuxPod            = "linuxptp-daemon"
+	ptpLinuxContainer      = "linuxptp-daemon-container"
+)
+
+func isClockSync(apiClient *clients.Settings) (bool, error) {
+
+	podList, err := pod.List(apiClient, machineConfigNamespace)
+	if err != nil {
+		return false, fmt.Errorf("failed to get Machine config pod list, %w", err)
+	}
+
+	SyncMessage := "System clock synchronized: yes"
+
+	for _, pod := range podList {
+		if strings.Contains(pod.Object.Name, machineConfigDaemonPod) {
+			synccmd := []string{"chroot", "/rootfs", "/bin/sh", "-c", "timedatectl"}
+			cmd, err := pod.ExecCommand(synccmd)
+
+			if (len(cmd.String()) == 0) || (err != nil) {
+				return false, fmt.Errorf("failed to check clock sync status from machine config container, %w, %s", err, cmd.String())
+			}
+
+			if !strings.Contains(cmd.String(), SyncMessage) {
+				return false, fmt.Errorf("clock not in sync, %w", err)
+			}
+
+			return true, nil
+		}
+	}
+
+	return false, fmt.Errorf("sync status could not be verified")
+}
+
+func isPtpClockSync(apiClient *clients.Settings) (bool, error) {
+
+	podList, err := pod.List(apiClient, ptpNamespace)
+	if err != nil {
+		return false, fmt.Errorf("failed to get PTP pod list, %w", err)
+	}
+
+	ptpSyncPattern := `openshift_ptp_clock_state{iface="CLOCK_REALTIME",node=".*",process="phc2sys"} 1`
+	ptpRe := regexp.MustCompile(ptpSyncPattern)
+
+	for _, pod := range podList {
+		if strings.Contains(pod.Object.Name, ptpLinuxPod) {
+			synccmd := []string{"curl", "-s", "http://localhost:9091/metrics"}
+			cmd, err := pod.ExecCommand(synccmd)
+
+			if (len(cmd.String()) == 0) || (err != nil) {
+				return false, fmt.Errorf("failed to check PTP sync status, %w, %s", err, cmd.String())
+			}
+			if !ptpRe.MatchString(cmd.String()) {
+				return false, fmt.Errorf("PTP not in sync, %s", cmd.String())
+			}
+
+			return true, nil
+		}
+
+	}
+
+	return false, fmt.Errorf("sync status could not be verified")
+}
+
+// ValidatePTPStatus checks the clock sync status and also checks the PTP logs.
+func ValidatePTPStatus(apiClient *clients.Settings, timeInterval time.Duration) (bool, error) {
+
+	clockSync, err := isClockSync(apiClient)
+	if err != nil {
+		return false, err
+	}
+
+	ptpSync, err := isPtpClockSync(apiClient)
+	if err != nil {
+		return false, err
+	}
+
+	ptpSync = ptpSync && clockSync
+
+	podList, err := pod.List(apiClient, ptpNamespace)
+	if err != nil {
+		return ptpSync, err
+	}
+
+	if len(podList) == 0 {
+		return ptpSync, fmt.Errorf("PTP logs don't exist")
+	}
+
+	var ptpLog string
+
+	for _, pod := range podList {
+		if strings.Contains(pod.Object.Name, ptpLinuxPod) {
+			ptpLog, err = pod.GetLog(timeInterval, ptpLinuxContainer)
+			if err != nil {
+				return ptpSync, err
+			}
+		}
+	}
+
+	switch {
+	case strings.Contains(ptpLog, "timed out while polling for tx timestamp"):
+		return ptpSync, fmt.Errorf("error: PTP timed out")
+	case strings.Contains(ptpLog, "jump"):
+		return ptpSync, fmt.Errorf("error: PTP jump")
+	case len(ptpLog) == 0:
+		return ptpSync, fmt.Errorf("error: PTP logs not found")
+	}
+
+	return ptpSync, nil
+}

tests/system-tests/internal/stability/stability.go

@@ -0,0 +1,171 @@
+package stability
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/openshift-kni/eco-goinfra/pkg/clients"
+	"github.com/openshift-kni/eco-goinfra/pkg/ocm"
+	"github.com/openshift-kni/eco-goinfra/pkg/pod"
+	"github.com/openshift-kni/eco-gotests/tests/system-tests/internal/ptp"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	runtimeclient "sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+func buildOutputLine(data map[string]string) string {
+	keys := make([]string, 0, len(data))
+	for key := range data {
+		keys = append(keys, key)
+	}
+
+	sort.Strings(keys)
+
+	var sb strings.Builder
+	for _, key := range keys {
+		sb.WriteString(fmt.Sprintf("%s,%s,", key, data[key]))
+	}
+
+	line := strings.TrimSuffix(sb.String(), ",")
+
+	currentTime := time.Now().Format(time.RFC3339)
+
+	return fmt.Sprintf("%s,%s\n", currentTime, line)
+}
+
+// SavePTPStatus stores the PTP status in the outputFile.
+func SavePTPStatus(apiClient *clients.Settings, outputFile string, timeInterval time.Duration) error {
+	file, err := os.OpenFile(outputFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	ptpOnSync, err := ptp.ValidatePTPStatus(apiClient, timeInterval)
+
+	currentTime := time.Now().Format(time.RFC3339)
+
+	var line string
+
+	if ptpOnSync {
+		line = fmt.Sprintf("%s,Sync,\"%v\"\n", currentTime, err)
+	} else {
+		line = fmt.Sprintf("%s,Unsync,\"%v\"\n", currentTime, err)
+	}
+
+	_, err = file.WriteString(line)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// SavePodsRestartsInNamespace stores the pod restarts of all pods in a namespace in the outputFile.
+func SavePodsRestartsInNamespace(apiClient *clients.Settings, namespace string, outputFile string) error {
+	podList, err := pod.List(apiClient, namespace, v1.ListOptions{})
+	if err != nil {
+		return err
+	}
+
+	file, err := os.OpenFile(outputFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	allPodsInNamespace := make(map[string]string)
+
+	for _, pod := range podList {
+		totalRestarts := 0
+		for _, containerStatus := range pod.Object.Status.ContainerStatuses {
+			totalRestarts += int(containerStatus.RestartCount)
+		}
+
+		allPodsInNamespace[pod.Object.Name] = fmt.Sprintf("%d", totalRestarts)
+	}
+
+	entry := buildOutputLine(allPodsInNamespace)
+
+	_, err = file.WriteString(entry)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// SavePolicyStatus stores the status of all policies in the outputFile.
+func SavePolicyStatus(apiClient *clients.Settings, clusterName string, outputFile string) error {
+	file, err := os.OpenFile(outputFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	allPolicies, err := ocm.ListPoliciesInAllNamespaces(apiClient,
+		runtimeclient.ListOptions{Namespace: clusterName})
+	if err != nil {
+		return fmt.Errorf("failed to get policies in %q NS: %w", clusterName, err)
+	}
+
+	allPoliciesStatus := make(map[string]string)
+
+	for _, policy := range allPolicies {
+		pName := policy.Definition.Name
+		pState := string(policy.Object.Status.ComplianceState)
+		allPoliciesStatus[pName] = pState
+	}
+
+	entry := buildOutputLine(allPoliciesStatus)
+
+	_, err = file.WriteString(entry)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// VerifyStabilityStatusChange checks if there has been a change between in
+// the column of the stability output file.
+func VerifyStabilityStatusChange(filePath string) (bool, error) {
+	file, err := os.OpenFile(filePath, os.O_RDONLY, 0)
+	if err != nil {
+		return false, err
+	}
+
+	defer file.Close()
+
+	var previousLine []string
+
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		line := scanner.Text()
+		columns := strings.Split(line, ",")
+
+		if previousLine == nil {
+			previousLine = columns
+
+			continue
+		}
+
+		for i := 1; i < len(columns); i += 2 {
+			if columns[i] != previousLine[i] || columns[i+1] != previousLine[i+1] {
+				return true, fmt.Errorf("change detected in column %d:\n previous: %s - %s\n current: %s - %s",
+					i+1, previousLine[i], previousLine[i+1], columns[i], columns[i+1])
+			}
+		}
+
+		previousLine = columns
+	}
+
+	if err := scanner.Err(); err != nil {
+		return false, err
+	}
+
+	return false, nil
+}

tests/system-tests/ran-du/internal/randuconfig/config.go

@@ -25,9 +25,13 @@ type RanDuConfig struct {
 		CreateShellCmd string `yaml:"create_shell_cmd" envconfig:"ECO_RANDU_TESTWORKLOAD_CREATE_SHELLCMD"`
 		DeleteShellCmd string `yaml:"delete_shell_cmd" envconfig:"ECO_RANDU_TESTWORKLOAD_DELETE_SHELLCMD"`
 	} `yaml:"randu_test_workload"`
-	LaunchWorkloadIterations int `yaml:"launch_workload_iterations" envconfig:"ECO_RANDU_LAUNCH_WORKLOAD_ITERATIONS"`
-	SoftRebootIterations     int `yaml:"soft_reboot_iterations" envconfig:"ECO_RANDU_SOFT_REBOOT_ITERATIONS"`
-	HardRebootIterations     int `yaml:"hard_reboot_iterations" envconfig:"ECO_RANDU_HARD_REBOOT_ITERATIONS"`
+	LaunchWorkloadIterations int    `yaml:"launch_workload_iterations" envconfig:"ECO_RANDU_LAUNCH_WORKLOAD_ITERATIONS"`
+	SoftRebootIterations     int    `yaml:"soft_reboot_iterations" envconfig:"ECO_RANDU_SOFT_REBOOT_ITERATIONS"`
+	HardRebootIterations     int    `yaml:"hard_reboot_iterations" envconfig:"ECO_RANDU_HARD_REBOOT_ITERATIONS"`
+	StabilityDurationMins    int64  `yaml:"stability_duration_mins" envconfig:"ECO_RANDU_STABILITY_DUR_MINS"`
+	StabilityIntervalMins    int64  `yaml:"stability_interval_mins" envconfig:"ECO_RANDU_STABILITY_INT_MINS"`
+	StabilityOutputPath      string `yaml:"stability_output_path" envconfig:"ECO_RANDU_STABILITY_OUTPUT_PATH"`
+	PtpEnabled               bool   `yaml:"ptp_enabled" envconfig:"ECO_RANDU_PTP_ENABLED"`
 }
 
 // NewRanDuConfig returns instance of RanDuConfig config type.

tests/system-tests/ran-du/internal/randuconfig/default.yaml

@@ -8,3 +8,9 @@ randu_test_workload:
 launch_workload_iterations: 5
 soft_reboot_iterations: 5
 hard_reboot_iterations: 5
+
+stability_duration_mins: 30
+stability_interval_mins: 5
+stability_output_path: "/tmp/reports"
+
+ptp_enabled: true