Verify the NooBaa DB PostreSQL version at upgrade

ocs_ci/helpers/helpers.py

@@ -29,10 +29,16 @@
     update_container_with_proxy_env,
 )
 from ocs_ci.ocs.utils import get_non_acm_cluster_config, get_pod_name_by_pattern
-from ocs_ci.ocs.utils import mirror_image
+from ocs_ci.ocs.utils import (
+    mirror_image,
+    get_expected_nb_db_psql_version,
+    get_nb_db_psql_version_from_image,
+    query_nb_db_psql_version,
+)
 from ocs_ci.ocs import constants, defaults, node, ocp, exceptions
 from ocs_ci.ocs.exceptions import (
     CommandFailed,
+    ResourceNotFoundError,
     ResourceWrongStatusException,
     TimeoutExpiredError,
     UnavailableBuildException,
@@ -5368,6 +5374,52 @@ def update_volsync_channel():
             )
 
 
+def verify_nb_db_psql_version(check_image_name_version=True):
+    """
+    Verify that the NooBaa DB PostgreSQL version matches the expectation
+    that is derived from the NooBaa CR.
+
+    Args:
+        check_image_name_version (bool): If True, also check that the
+                                         version from the name of the image
+                                         in the NooBaa DB Statefulset
+                                         matches the one queried from the DB.
+
+    Raises:
+        AssertionError: If the NooBaa DB PostgreSQL version doesn't match the
+                        NooBaa CR expectation.
+        UnexpectedBehaviour: If the parsing or extraction of the versions fails
+                             due to changes in the CRs.
+    """
+
+    try:
+        expected_version = version.get_semantic_version(
+            get_expected_nb_db_psql_version(), only_major=True
+        )
+        version_from_query = version.get_semantic_version(
+            query_nb_db_psql_version(), only_major=True
+        )
+
+        if check_image_name_version:
+            version_from_image = version.get_semantic_version(
+                get_nb_db_psql_version_from_image(), only_major=True
+            )
+            assert version_from_image == version_from_query, (
+                f"NooBaa DB PostgreSQL version mismatch between the image and the DB query. "
+                f"Image: {version_from_image}, Query: {version_from_query}"
+            )
+
+        assert version_from_query == expected_version, (
+            f"NooBaa DB PostgreSQL version doesn't match the NooBaa CR expectation. "
+            f"Expected version: {expected_version}, Actual version: {version_from_query}"
+        )
+
+    except ResourceNotFoundError:
+        logger.warning(
+            "NooBaa DB PostgreSQL version couldn't be verified as one or more resources are missing."
+        )
+
+
 def verify_performance_profile_change(perf_profile):
     """
     Verify that newly applied performance profile got updated in storage cluster

ocs_ci/ocs/ocs_upgrade.py

@@ -754,6 +754,10 @@ def run_ocs_upgrade(
         upgrade_ocs.version_before_upgrade,
     )
 
+    from ocs_ci.helpers.helpers import verify_nb_db_psql_version
+
+    verify_nb_db_psql_version()
+
     # update external secrets
     if config.DEPLOYMENT["external_mode"]:
         upgrade_version = version.get_semantic_version(upgrade_version, True)

ocs_ci/ocs/utils.py

@@ -24,8 +24,13 @@
 from ocs_ci.ocs import constants
 from ocs_ci.ocs.external_ceph import RolesContainer, Ceph, CephNode
 from ocs_ci.ocs.clients import WinNode
-from ocs_ci.ocs.exceptions import CommandFailed, ExternalClusterDetailsException
-from ocs_ci.ocs.ocp import OCP
+from ocs_ci.ocs.exceptions import (
+    CommandFailed,
+    ExternalClusterDetailsException,
+    ResourceNotFoundError,
+    UnexpectedBehaviour,
+)
+from ocs_ci.ocs.ocp import OCP, get_images
 from ocs_ci.ocs.openstack import CephVMNode
 from ocs_ci.ocs.parallel import parallel
 from ocs_ci.ocs.resources.ocs import OCS
@@ -34,6 +39,7 @@
 from ocs_ci.utility.retry import retry
 from ocs_ci.utility.utils import (
     create_directory_path,
+    exec_nb_db_query,
     mirror_image,
     run_cmd,
     get_oadp_version,
@@ -1890,3 +1896,88 @@ def get_dr_operator_versions():
             if submariner_operator_version:
                 versions_dic["submariner_version"] = submariner_operator_version
     return versions_dic
+
+
+def get_nb_db_psql_version_from_image():
+    """
+    Get the NooBaa DB PostgreSQL version from the image name in the NooBaa DB statefulset
+
+    Returns:
+       str: The NooBaa DB PostgreSQL version
+
+    Raises:
+        ResourceNotFoundError: If the NooBaa DB statefulset is not available
+        UnexpectedBehaviour: If the NooBaa DB version could not be extracted from the
+                             NooBaa DB statefulset image
+
+    """
+    nb_db_sts_obj = OCP(
+        kind=constants.STATEFULSET,
+        namespace=ocsci_config.ENV_DATA["cluster_namespace"],
+        resource_name=constants.NOOBAA_DB_STATEFULSET,
+    )
+    if not nb_db_sts_obj.data:
+        raise ResourceNotFoundError(
+            f"NooBaa DB statefulset {constants.NOOBAA_DB_STATEFULSET} not found"
+        )
+
+    try:
+        images = get_images(nb_db_sts_obj.data)
+        psql_img = next(iter(images.values()))  # Only one image is expected
+        re_match = re.search(r"postgresql-(\d+(\.\d+)*)", psql_img)
+        return re_match.group(1)
+    except Exception as e:
+        raise UnexpectedBehaviour(
+            f"Failed to extract the NooBaa DB version from its stateful set: {e}"
+        )
+
+
+def query_nb_db_psql_version():
+    """
+    Query the NooBaa DB for its PostgreSQL version
+
+    Returns:
+        str: The NooBaa DB version
+
+    Raises:
+        ResourceNotFoundError: If the NooBaa DB pod was not found
+        UnexpectedBehaviour: If the NooBaa DB version could not be extracted from the
+                             NooBaa DB pod
+    """
+    try:
+        raw_output = exec_nb_db_query("SELECT version();")[0]
+    except IndexError:
+        raise UnexpectedBehaviour("Failed to query the NooBaa DB for its version")
+    return re.search(r"PostgreSQL (\S+)", raw_output).group(1)
+
+
+def get_expected_nb_db_psql_version():
+    """
+    Get the expected NooBaa DB version from the NooBaa CR
+
+    Returns:
+        str: The expected NooBaa DB version
+
+    Raises:
+        ResourceNotFoundError: If the NooBaa CR was not found
+        UnexpectedBehaviour: If the NooBaa DB version could not be extracted from the
+    """
+
+    nb_cr_obj = OCP(
+        kind=constants.NOOBAA_RESOURCE_NAME,
+        namespace=ocsci_config.ENV_DATA["cluster_namespace"],
+        resource_name=constants.NOOBAA_RESOURCE_NAME,
+    )
+    if not nb_cr_obj:
+        raise ResourceNotFoundError(
+            f"NooBaa CR {constants.NOOBAA_RESOURCE_NAME} not found"
+        )
+
+    try:
+        psql_image = nb_cr_obj.data["spec"]["dbImage"]
+        re_match = re.search(r"postgresql-(\d+(\.\d+)*)", psql_image)
+        return re_match.group(1)
+    except Exception as e:
+        raise UnexpectedBehaviour(
+            f"Failed to extract the NooBaa DB version from the NooBaa CR: {e}"
+        )

ocs_ci/utility/utils.py

@@ -46,6 +46,7 @@
     ClientDownloadError,
     CommandFailed,
     ConfigurationError,
+    ResourceNotFoundError,
     TagNotFoundException,
     TimeoutException,
     TimeoutExpiredError,
@@ -4982,16 +4983,25 @@ def exec_nb_db_query(query):
     Returns:
         list of str: The query result rows
 
+    Raises:
+        ResourceNotFoundError: If no NooBaa DB pod is found
+
     """
     # importing here to avoid circular imports
     from ocs_ci.ocs.resources import pod
 
-    nb_db_pod = pod.Pod(
-        **pod.get_pods_having_label(
-            label=constants.NOOBAA_DB_LABEL_47_AND_ABOVE,
-            namespace=config.ENV_DATA["cluster_namespace"],
-        )[0]
-    )
+    try:
+        nb_db_pod = pod.Pod(
+            **pod.get_pods_having_label(
+                label=constants.NOOBAA_DB_LABEL_47_AND_ABOVE,
+                namespace=config.ENV_DATA["cluster_namespace"],
+            )[0]
+        )
+    except IndexError:
+        raise ResourceNotFoundError(
+            f"The NooBaa DB pod with label {constants.NOOBAA_DB_LABEL_47_AND_ABOVE} "
+            f"was not found in namespace {config.ENV_DATA['cluster_namespace']}"
+        )
 
     response = nb_db_pod.exec_cmd_on_pod(
         command=f'psql -U postgres -d nbcore -c "{query}"',

tests/libtest/test_nb_db_psql_version.py

@@ -0,0 +1,15 @@
+import pytest
+from ocs_ci.framework.testlib import libtest
+from ocs_ci.helpers.helpers import verify_nb_db_psql_version
+
+
+@libtest
+@pytest.mark.parametrize(
+    "check_image",
+    [
+        False,
+        True,
+    ],
+)
+def test_nb_db_psql_version(check_image):
+    verify_nb_db_psql_version(check_image_name_version=check_image)