ibu cnf: post upgrade validations

Additional validations to run post ibu for cnf scenarios:
- validate upgraded cluster hostname is not the same as seed
- validate upgraded cluster network configuration
- validate no seed hostname references in pod logs
- validate no seed hostname references in etcd
- validate pre-upgrade DU configs are present after upgrade
- validate test workload pods are running without errors
- validate pre-upgrade files written to PVs are preserved during upgrade
- validate no images are being pulled after upgrade
- validate no seed references in the exported metrics
- validate no seed references in the exported logs

tests/lca/imagebasedupgrade/cnf/internal/cnfclusterinfo/cnfclusterinfo.go

@@ -2,10 +2,15 @@ package cnfclusterinfo
 
 import (
 	"errors"
+	"strings"
 
 	"github.com/golang/glog"
+	"github.com/openshift-kni/eco-goinfra/pkg/deployment"
 	"github.com/openshift-kni/eco-goinfra/pkg/nodes"
 	"github.com/openshift-kni/eco-goinfra/pkg/olm"
+	"github.com/openshift-kni/eco-goinfra/pkg/pod"
+	"github.com/openshift-kni/eco-goinfra/pkg/sriov"
+	"github.com/openshift-kni/eco-goinfra/pkg/statefulset"
 	"github.com/openshift-kni/eco-gotests/tests/internal/cluster"
 	"github.com/openshift-kni/eco-gotests/tests/lca/imagebasedupgrade/cnf/internal/cnfinittools"
 	"github.com/openshift-kni/eco-gotests/tests/lca/imagebasedupgrade/cnf/internal/cnfparams"
@@ -20,13 +25,37 @@ var (
 	PostUpgradeClusterInfo = ClusterStruct{}
 )
 
-// ClusterStruct is a struct that holds the cluster version and id.
+// WorkloadObjects is a struct that holds the workload objects.
+type WorkloadObjects struct {
+	Deployment  []string
+	StatefulSet []string
+}
+
+// WorkloadStruct is a struct that holds the workload info.
+type WorkloadStruct struct {
+	Namespace string
+	Objects   WorkloadObjects
+}
+
+// WorkloadPV struct holds the information to test that persitent volume content is not lost during upgrade.
+type WorkloadPV struct {
+	Namespace string
+	PodName   string
+	FilePath  string
+	Digest    string
+}
+
+// ClusterStruct is a struct that holds the cluster info pre and post upgrade.
 type ClusterStruct struct {
-	Version   string
-	ID        string
-	Name      string
-	Operators []string
-	NodeName  string
+	Version                  string
+	ID                       string
+	Name                     string
+	Operators                []string
+	NodeName                 string
+	SriovNetworks            []string
+	SriovNetworkNodePolicies []string
+	WorkloadResources        []WorkloadStruct
+	WorkloadPVs              WorkloadPV
 }
 
 // SaveClusterInfo is a dedicated func to save cluster info.
@@ -75,11 +104,87 @@ func (upgradeVar *ClusterStruct) SaveClusterInfo() error {
 		return errors.New("node list is empty")
 	}
 
+	sriovNet, err := sriov.List(cnfinittools.TargetSNOAPIClient, cnfinittools.CNFConfig.SriovOperatorNamespace)
+	if err != nil {
+		return err
+	}
+
+	for _, net := range sriovNet {
+		upgradeVar.SriovNetworks = append(upgradeVar.SriovNetworks, net.Object.Name)
+	}
+
+	sriovPolicy, err := sriov.ListPolicy(cnfinittools.TargetSNOAPIClient, cnfinittools.CNFConfig.SriovOperatorNamespace)
+
+	for _, policy := range sriovPolicy {
+		upgradeVar.SriovNetworkNodePolicies = append(upgradeVar.SriovNetworkNodePolicies, policy.Object.Name)
+	}
+
 	upgradeVar.Version = clusterVersion.Object.Status.Desired.Version
 	upgradeVar.ID = string(clusterVersion.Object.Spec.ClusterID)
 	upgradeVar.Name = targetSnoClusterName
 	upgradeVar.Operators = installedCSV
 	upgradeVar.NodeName = node[0].Object.Name
 
+	_ = upgradeVar.getWorkloadInfo()
+
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (upgradeVar *ClusterStruct) getWorkloadInfo() error {
+	for _, workloadNS := range strings.Split(cnfinittools.CNFConfig.IbuWorkloadNS, ",") {
+		workloadMap := WorkloadStruct{
+			Namespace: workloadNS,
+			Objects: WorkloadObjects{
+				Deployment:  []string{},
+				StatefulSet: []string{},
+			},
+		}
+
+		deployments, err := deployment.List(cnfinittools.TargetSNOAPIClient, workloadNS)
+		if err != nil {
+			return err
+		}
+
+		for _, deployment := range deployments {
+			workloadMap.Objects.Deployment = append(workloadMap.Objects.Deployment, deployment.Definition.Name)
+		}
+
+		statefulsets, err := statefulset.List(cnfinittools.TargetSNOAPIClient, workloadNS)
+		if err != nil {
+			return err
+		}
+
+		for _, statefulset := range statefulsets {
+			workloadMap.Objects.StatefulSet = append(workloadMap.Objects.StatefulSet, statefulset.Definition.Name)
+		}
+
+		upgradeVar.WorkloadResources = append(upgradeVar.WorkloadResources, workloadMap)
+	}
+
+	pod, err := pod.Pull(
+		cnfinittools.TargetSNOAPIClient,
+		cnfinittools.CNFConfig.IbuWorkloadPVPod,
+		cnfinittools.CNFConfig.IbuWorkloadPVNS,
+	)
+	if err != nil {
+		return err
+	}
+
+	cmd := []string{"bash", "-c", "md5sum " + cnfinittools.CNFConfig.IbuWorkloadPVFilePath + " ||true"}
+	getDigest, err := pod.ExecCommand(cmd)
+
+	if err != nil {
+		return err
+	}
+
+	upgradeVar.WorkloadPVs.Namespace = cnfinittools.CNFConfig.IbuWorkloadPVNS
+	upgradeVar.WorkloadPVs.PodName = cnfinittools.CNFConfig.IbuWorkloadPVPod
+	upgradeVar.WorkloadPVs.FilePath = cnfinittools.CNFConfig.IbuWorkloadPVFilePath
+	upgradeVar.WorkloadPVs.Digest = getDigest.String()
+
 	return nil
 }

tests/lca/imagebasedupgrade/cnf/internal/cnfconfig/config.go

@@ -20,9 +20,16 @@ const (
 // CNFConfig type contains cnf configuration.
 type CNFConfig struct {
 	*ibuconfig.IBUConfig
-	IBUWorkloadImage    string `yaml:"ibu_workload_image" envconfig:"ECO_LCA_IBU_CNF_WORKLOAD_IMAGE"`
-	TargetHubKubeConfig string `envconfig:"ECO_LCA_IBU_CNF_KUBECONFIG_TARGET_HUB"`
-	TargetSNOKubeConfig string `envconfig:"ECO_LCA_IBU_CNF_KUBECONFIG_TARGET_SNO"`
+	IBUWorkloadImage      string `yaml:"ibu_workload_image" envconfig:"ECO_LCA_IBU_CNF_WORKLOAD_IMAGE"`
+	TargetHubKubeConfig   string `envconfig:"ECO_LCA_IBU_CNF_KUBECONFIG_TARGET_HUB"`
+	TargetSNOKubeConfig   string `envconfig:"ECO_LCA_IBU_CNF_KUBECONFIG_TARGET_SNO"`
+	IbuWorkloadNS         string `yaml:"ibu_workload_validation_ns" envconfig:"ECO_LCA_IBU_CNF_WORKLOAD_NS"`
+	IbuWorkloadPVNS       string `yaml:"ibu_workload_validation_pv_ns" envconfig:"ECO_LCA_IBU_CNF_WORKLOAD_PV_NS"`
+	IbuWorkloadPVPod      string `yaml:"ibu_workload_validation_pv_pod_name" envconfig:"ECO_LCA_IBU_CNF_WORKLOAD_PV_POD"`
+	IbuWorkloadPVFilePath string `yaml:"ibu_workload_validation_pv_file_path" envconfig:"ECO_LCA_IBU_CNF_WORKLOAD_PV_FILE"`
+	IbuKcatImage          string `yaml:"ibu_kcat_image" envconfig:"ECO_LCA_IBU_CNF_KCAT_IMAGE"`
+	IbuKcatBroker         string `yaml:"ibu_kcat_broker" envconfig:"ECO_LCA_IBU_CNF_KCAT_BROKER"`
+	IbuKcatTopic          string `yaml:"ibu_kcat_topic" envconfig:"ECO_LCA_IBU_CNF_KCAT_TOPIC"`
 }
 
 // NewCNFConfig returns instance of CNFConfig type.

tests/lca/imagebasedupgrade/cnf/internal/cnfconfig/default.yaml

@@ -1,4 +1,10 @@
 ---
 # General configurations
 ibu_workload_image: "registry.redhat.io/openshift4/ose-hello-openshift-rhel8@sha256:10dca31348f07e1bfb56ee93c324525cceefe27cb7076b23e42ac181e4d1863e"
-...
+ibu_workload_validation_ns: "test"
+ibu_workload_validation_pv_ns: "test"
+ibu_workload_validation_pv_pod_name: "test10-0"
+ibu_workload_validation_pv_file_path: "/data/test10-0"
+ibu_kcat_image: "quay.io/ocp-edge-qe/kcat"
+ibu_kcat_broker: "kafka.example.com:9092"
+ibu_kcat_topic: "vran-qe"

tests/lca/imagebasedupgrade/cnf/internal/cnfparams/cnfvars.go

@@ -1,10 +1,14 @@
 package cnfparams
 
 import (
+	"time"
+
 	"github.com/openshift-kni/eco-gotests/tests/lca/imagebasedupgrade/internal/ibuparams"
 )
 
 var (
 	// Labels represents the range of labels that can be used for test cases selection.
 	Labels = []string{ibuparams.Label, Label}
+	// WorkloadReadyTimeout represents the timeout for the workload to be ready.
+	WorkloadReadyTimeout = 5 * time.Minute
 )