Check for Kube Object Sync time before running failover for Discovered Apps

ocs_ci/helpers/dr_helpers.py

@@ -1715,3 +1715,49 @@ def verify_drpc_deletion(cmd, expected_output_lst):
         if expected_output not in drpc_out.stderr.decode():
             return False
     return True
+
+
+def verify_last_kubeobject_protection_time(drpc_obj, kubeobject_sync_interval):
+    """
+    Verifies that the lastKubeObjectProtectionTime for a given DRPC object is within the expected range.
+
+    Args:
+        drpc_obj (obj): DRPC object
+        kubeobject_sync_interval (int): The KubeObject sync interval in minutes
+
+    Returns:
+        str: Current lastKubeObjectProtectionTime
+
+    Raises:
+        AssertionError: If the lastKubeObjectProtectionTime is outside the expected range
+            (greater than or equal to two times the scheduling interval)
+
+    """
+    restore_index = config.cur_index
+    config.switch_acm_ctx()
+    last_kubeobject_protection_time = drpc_obj.get_last_kubeobject_protection_time()
+    if not last_kubeobject_protection_time:
+        assert last_kubeobject_protection_time, (
+            "There is no lastKubeObjectProtectionTime. "
+            "Verify that certificates are included correctly in the Ramen Hub configuration map."
+        )
+    # Verify lastGroupSyncTime
+    time_format = "%Y-%m-%dT%H:%M:%SZ"
+    last_kubeobject_protection_time_formatted = datetime.strptime(
+        last_kubeobject_protection_time, time_format
+    )
+    current_time = datetime.strptime(
+        datetime.utcnow().strftime(time_format), time_format
+    )
+    time_since_last_sync = (
+        current_time - last_kubeobject_protection_time_formatted
+    ).total_seconds() / 60
+    logger.info(
+        f"Time in minutes since the last Kube Object sync {time_since_last_sync}"
+    )
+    assert (
+        time_since_last_sync < 2 * kubeobject_sync_interval
+    ), "The syncing of Kube Resources is exceeding three times the Kube object sync interval"
+    logger.info("Verified lastKubeObjectProtectionTime value within expected range")
+    config.switch_ctx(restore_index)
+    return last_kubeobject_protection_time

ocs_ci/ocs/dr/dr_workload.py

@@ -1076,7 +1076,8 @@ def __init__(self, **kwargs):
         self.discovered_apps_placement_name = kwargs.get("workload_placement_name")
         self.drpc_yaml_file = os.path.join(constants.DRPC_PATH)
         self.placement_yaml_file = os.path.join(constants.PLACEMENT_PATH)
-        self.kubeobject_capture_interval = f"{generate_kubeobject_capture_interval()}m"
+        self.kubeobject_capture_interval_int = generate_kubeobject_capture_interval()
+        self.kubeobject_capture_interval = f"{self.kubeobject_capture_interval_int}m"
         self.protection_type = kwargs.get("protection_type")
         self.target_clone_dir = config.ENV_DATA.get(
             "target_clone_dir", constants.DR_WORKLOAD_REPO_BASE_DIR

ocs_ci/ocs/resources/drpc.py

@@ -101,6 +101,22 @@ def get_last_group_sync_time(self):
         logger.info(f"Current lastGroupSyncTime is {last_group_sync_time}.")
         return last_group_sync_time
 
+    def get_last_kubeobject_protection_time(self):
+        """
+        Fetch lastKubeObjectProtectionTime from DRPC
+
+        Returns:
+            str: lastKubeObjectProtectionTime
+
+        """
+        last_kubeobject_protection_time = (
+            self.get().get("status").get("lastKubeObjectProtectionTime")
+        )
+        logger.info(
+            f"Current lastKubeObjectProtectionTime is {last_kubeobject_protection_time}."
+        )
+        return last_kubeobject_protection_time
+
 
 def get_drpc_name(namespace, switch_ctx=None):
     """

tests/functional/disaster-recovery/regional-dr/test_failover_and_relocate_discovered_apps.py

@@ -3,10 +3,11 @@
 
 
 from ocs_ci.framework import config
-from ocs_ci.framework.testlib import acceptance, tier1
+from ocs_ci.framework.testlib import acceptance, tier1, skipif_ocs_version
 from ocs_ci.framework.pytest_customization.marks import rdr, turquoise_squad
 from ocs_ci.helpers import dr_helpers
-
+from ocs_ci.ocs import constants
+from ocs_ci.ocs.resources.drpc import DRPC
 
 logger = logging.getLogger(__name__)
 
@@ -15,6 +16,7 @@
 @acceptance
 @tier1
 @turquoise_squad
+@skipif_ocs_version("<4.16")
 class TestFailoverAndRelocateWithDiscoveredApps:
     """
     Test Failover and Relocate with Discovered Apps
@@ -45,10 +47,16 @@ def test_failover_and_relocate_discovered_apps(self, discovered_apps_dr_workload
         scheduling_interval = dr_helpers.get_scheduling_interval(
             rdr_workload.workload_namespace, discovered_apps=True
         )
+        drpc_obj = DRPC(namespace=constants.DR_OPS_NAMESAPCE)
         wait_time = 2 * scheduling_interval  # Time in minutes
         logger.info(f"Waiting for {wait_time} minutes to run IOs")
         sleep(wait_time * 60)
 
+        logger.info("Checking for lastKubeObjectProtectionTime")
+        dr_helpers.verify_last_kubeobject_protection_time(
+            drpc_obj, rdr_workload.kubeobject_capture_interval_int
+        )
+
         dr_helpers.failover(
             failover_cluster=secondary_cluster_name,
             namespace=rdr_workload.workload_namespace,
@@ -87,11 +95,17 @@ def test_failover_and_relocate_discovered_apps(self, discovered_apps_dr_workload
         scheduling_interval = dr_helpers.get_scheduling_interval(
             rdr_workload.workload_namespace, discovered_apps=True
         )
+
         logger.info("Running Relocate Steps")
         wait_time = 2 * scheduling_interval  # Time in minutes
         logger.info(f"Waiting for {wait_time} minutes to run IOs")
         sleep(wait_time * 60)
 
+        logger.info("Checking for lastKubeObjectProtectionTime")
+        dr_helpers.verify_last_kubeobject_protection_time(
+            drpc_obj, rdr_workload.kubeobject_capture_interval_int
+        )
+
         dr_helpers.relocate(
             preferred_cluster=secondary_cluster_name,
             namespace=rdr_workload.workload_namespace,
@@ -101,4 +115,9 @@ def test_failover_and_relocate_discovered_apps(self, discovered_apps_dr_workload
             workload_instance=rdr_workload,
         )
 
+        logger.info("Checking for lastKubeObjectProtectionTime post Relocate Operation")
+        dr_helpers.verify_last_kubeobject_protection_time(
+            drpc_obj, rdr_workload.kubeobject_capture_interval_int
+        )
+
         # TODO: Add data integrity checks