Verify the nb db psql version at upgrade

Signed-off-by: Sagi Hirshfeld <[email protected]>

ocs_ci/helpers/helpers.py

@@ -29,10 +29,16 @@
     update_container_with_proxy_env,
 )
 from ocs_ci.ocs.utils import get_non_acm_cluster_config, get_pod_name_by_pattern
-from ocs_ci.ocs.utils import mirror_image
+from ocs_ci.ocs.utils import (
+    mirror_image,
+    get_expected_nb_db_psql_version,
+    get_nb_db_psql_version_from_image,
+    query_nb_db_psql_version,
+)
 from ocs_ci.ocs import constants, defaults, node, ocp, exceptions
 from ocs_ci.ocs.exceptions import (
     CommandFailed,
+    ResourceNotFoundError,
     ResourceWrongStatusException,
     TimeoutExpiredError,
     UnavailableBuildException,
@@ -818,9 +824,9 @@ def create_storage_class(
     sc_data["metadata"]["namespace"] = config.ENV_DATA["cluster_namespace"]
     for key in ["node-stage", "provisioner", "controller-expand"]:
         sc_data["parameters"][f"csi.storage.k8s.io/{key}-secret-name"] = secret_name
-        sc_data["parameters"][
-            f"csi.storage.k8s.io/{key}-secret-namespace"
-        ] = config.ENV_DATA["cluster_namespace"]
+        sc_data["parameters"][f"csi.storage.k8s.io/{key}-secret-namespace"] = (
+            config.ENV_DATA["cluster_namespace"]
+        )
 
     if annotations:
         sc_data["metadata"]["annotations"] = annotations
@@ -2983,12 +2989,12 @@ def collect_performance_stats(dir_name):
 
     performance_stats["master_node_utilization"] = master_node_utilization_from_adm_top
     performance_stats["worker_node_utilization"] = worker_node_utilization_from_adm_top
-    performance_stats[
-        "master_node_utilization_from_oc_describe"
-    ] = master_node_utilization_from_oc_describe
-    performance_stats[
-        "worker_node_utilization_from_oc_describe"
-    ] = worker_node_utilization_from_oc_describe
+    performance_stats["master_node_utilization_from_oc_describe"] = (
+        master_node_utilization_from_oc_describe
+    )
+    performance_stats["worker_node_utilization_from_oc_describe"] = (
+        worker_node_utilization_from_oc_describe
+    )
 
     file_name = os.path.join(log_dir_path, "performance")
     with open(file_name, "w") as outfile:
@@ -4951,9 +4957,9 @@ def configure_node_network_configuration_policy_on_all_worker_nodes():
         node_network_configuration_policy["spec"]["nodeSelector"][
             "kubernetes.io/hostname"
         ] = worker_node_name
-        node_network_configuration_policy["metadata"][
-            "name"
-        ] = worker_network_configuration["node_network_configuration_policy_name"]
+        node_network_configuration_policy["metadata"]["name"] = (
+            worker_network_configuration["node_network_configuration_policy_name"]
+        )
         node_network_configuration_policy["spec"]["desiredState"]["interfaces"][0][
             "ipv4"
         ]["address"][0]["ip"] = worker_network_configuration[
@@ -5366,3 +5372,49 @@ def update_volsync_channel():
             logger.error(
                 f"Pod volsync-controller-manager not in {constants.STATUS_RUNNING} after 300 seconds"
             )
+
+
+def verify_nb_db_psql_version(check_image_name_version=True):
+    """
+    Verify that the NooBaa DB PostgreSQL version matches the expectation
+    that is derived from the NooBaa CR.
+
+    Args:
+        check_image_name_version (bool): If True, also check that the
+                                         version from the name of the image
+                                         in the NooBaa DB Statefulset
+                                         matches the one queried from the DB.
+
+    Raises:
+        AssertionError: If the NooBaa DB PostgreSQL version doesn't match the
+                        NooBaa CR expectation.
+        UnexpectedBehaviour: If the parsing or extraction of the versions fails
+                             due to changes in the CRs.
+    """
+
+    try:
+        expected_version = version.get_semantic_version(
+            get_expected_nb_db_psql_version(), only_major=True
+        )
+        version_from_query = version.get_semantic_version(
+            query_nb_db_psql_version(), only_major=True
+        )
+
+        if check_image_name_version:
+            version_from_image = version.get_semantic_version(
+                get_nb_db_psql_version_from_image(), only_major=True
+            )
+            assert version_from_image == version_from_query, (
+                f"NooBaa DB PostgreSQL version mismatch between the image and the DB query. "
+                f"Image: {version_from_image}, Query: {version_from_query}"
+            )
+
+        assert version_from_query == expected_version, (
+            f"NooBaa DB PostgreSQL version doesn't match the NooBaa CR expectation. "
+            f"Expected version: {expected_version}, Actual version: {version_from_query}"
+        )
+
+    except ResourceNotFoundError:
+        logger.warning(
+            "NooBaa DB PostgreSQL version couldn't be verified as one or more resources are missing."
+        )

ocs_ci/ocs/ocs_upgrade.py

@@ -754,6 +754,10 @@ def run_ocs_upgrade(
         upgrade_ocs.version_before_upgrade,
     )
 
+    from ocs_ci.helpers.helpers import verify_nb_db_psql_version
+
+    verify_nb_db_psql_version()
+
     # update external secrets
     if config.DEPLOYMENT["external_mode"]:
         upgrade_version = version.get_semantic_version(upgrade_version, True)

ocs_ci/ocs/utils.py

@@ -23,8 +23,13 @@
 from ocs_ci.ocs import constants
 from ocs_ci.ocs.external_ceph import RolesContainer, Ceph, CephNode
 from ocs_ci.ocs.clients import WinNode
-from ocs_ci.ocs.exceptions import CommandFailed, ExternalClusterDetailsException
-from ocs_ci.ocs.ocp import OCP
+from ocs_ci.ocs.exceptions import (
+    CommandFailed,
+    ExternalClusterDetailsException,
+    ResourceNotFoundError,
+    UnexpectedBehaviour,
+)
+from ocs_ci.ocs.ocp import OCP, get_images
 from ocs_ci.ocs.openstack import CephVMNode
 from ocs_ci.ocs.parallel import parallel
 from ocs_ci.ocs.resources.ocs import OCS
@@ -33,6 +38,7 @@
 from ocs_ci.utility.retry import retry
 from ocs_ci.utility.utils import (
     create_directory_path,
+    exec_nb_db_query,
     mirror_image,
     run_cmd,
     get_oadp_version,
@@ -1863,3 +1869,88 @@ def get_dr_operator_versions():
             if submariner_operator_version:
                 versions_dic["submariner_version"] = submariner_operator_version
     return versions_dic
+
+
+def get_nb_db_psql_version_from_image():
+    """
+    Get the NooBaa DB PostgreSQL version from the image name in the NooBaa DB statefulset
+
+    Returns:
+       str: The NooBaa DB PostgreSQL version
+
+    Raises:
+        ResourceNotFoundError: If the NooBaa DB statefulset is not available
+        UnexpectedBehaviour: If the NooBaa DB version could not be extracted from the
+                             NooBaa DB statefulset image
+
+    """
+    nb_db_sts_obj = OCP(
+        kind=constants.STATEFULSET,
+        namespace=ocsci_config.ENV_DATA["cluster_namespace"],
+        resource_name=constants.NOOBAA_DB_STATEFULSET,
+    )
+    if not nb_db_sts_obj.data:
+        raise ResourceNotFoundError(
+            f"NooBaa DB statefulset {constants.NOOBAA_DB_STATEFULSET} not found"
+        )
+
+    try:
+        images = get_images(nb_db_sts_obj.data)
+        psql_img = next(iter(images.values()))  # Only one image is expected
+        re_match = re.search(r"postgresql-(\d+(\.\d+)*)", psql_img)
+        return re_match.group(1)
+    except Exception as e:
+        raise UnexpectedBehaviour(
+            f"Failed to extract the NooBaa DB version from its stateful set: {e}"
+        )
+
+
+def query_nb_db_psql_version():
+    """
+    Query the NooBaa DB for its PostgreSQL version
+
+    Returns:
+        str: The NooBaa DB version
+
+    Raises:
+        ResourceNotFoundError: If the NooBaa DB pod was not found
+        UnexpectedBehaviour: If the NooBaa DB version could not be extracted from the
+                             NooBaa DB pod
+    """
+    try:
+        raw_output = exec_nb_db_query("SELECT version();")[0]
+    except IndexError:
+        raise UnexpectedBehaviour("Failed to query the NooBaa DB for its version")
+    return re.search(r"PostgreSQL (\S+)", raw_output).group(1)
+
+
+def get_expected_nb_db_psql_version():
+    """
+    Get the expected NooBaa DB version from the NooBaa CR
+
+    Returns:
+        str: The expected NooBaa DB version
+
+    Raises:
+        ResourceNotFoundError: If the NooBaa CR was not found
+        UnexpectedBehaviour: If the NooBaa DB version could not be extracted from the
+    """
+
+    nb_cr_obj = OCP(
+        kind=constants.NOOBAA_RESOURCE_NAME,
+        namespace=ocsci_config.ENV_DATA["cluster_namespace"],
+        resource_name=constants.NOOBAA_RESOURCE_NAME,
+    )
+    if not nb_cr_obj:
+        raise ResourceNotFoundError(
+            f"NooBaa CR {constants.NOOBAA_RESOURCE_NAME} not found"
+        )
+
+    try:
+        psql_image = nb_cr_obj.data["spec"]["dbImage"]
+        re_match = re.search(r"postgresql-(\d+(\.\d+)*)", psql_image)
+        return re_match.group(1)
+    except Exception as e:
+        raise UnexpectedBehaviour(
+            f"Failed to extract the NooBaa DB version from the NooBaa CR: {e}"
+        )

ocs_ci/utility/utils.py

@@ -46,6 +46,7 @@
     ClientDownloadError,
     CommandFailed,
     ConfigurationError,
+    ResourceNotFoundError,
     TagNotFoundException,
     TimeoutException,
     TimeoutExpiredError,
@@ -4981,16 +4982,25 @@ def exec_nb_db_query(query):
     Returns:
         list of str: The query result rows
 
+    Raises:
+        ResourceNotFoundError: If no NooBaa DB pod is found
+
     """
     # importing here to avoid circular imports
     from ocs_ci.ocs.resources import pod
 
-    nb_db_pod = pod.Pod(
-        **pod.get_pods_having_label(
-            label=constants.NOOBAA_DB_LABEL_47_AND_ABOVE,
-            namespace=config.ENV_DATA["cluster_namespace"],
-        )[0]
-    )
+    try:
+        nb_db_pod = pod.Pod(
+            **pod.get_pods_having_label(
+                label=constants.NOOBAA_DB_LABEL_47_AND_ABOVE,
+                namespace=config.ENV_DATA["cluster_namespace"],
+            )[0]
+        )
+    except IndexError:
+        raise ResourceNotFoundError(
+            f"The NooBaa DB pod with label {constants.NOOBAA_DB_LABEL_47_AND_ABOVE} "
+            f"was not found in namespace {config.ENV_DATA['cluster_namespace']}"
+        )
 
     response = nb_db_pod.exec_cmd_on_pod(
         command=f'psql -U postgres -d nbcore -c "{query}"',

tests/libtest/test_nb_db_psql_version.py

@@ -0,0 +1,15 @@
+import pytest
+from ocs_ci.framework.testlib import libtest
+from ocs_ci.helpers.helpers import verify_nb_db_psql_version
+
+
+@libtest
+@pytest.mark.parametrize(
+    "check_image",
+    [
+        False,
+        True,
+    ],
+)
+def test_nb_db_psql_version(check_image):
+    verify_nb_db_psql_version(check_image_name_version=check_image)