diff --git a/ocs_ci/helpers/dr_helpers.py b/ocs_ci/helpers/dr_helpers.py index 32a6c96a1b7..e3df184247b 100644 --- a/ocs_ci/helpers/dr_helpers.py +++ b/ocs_ci/helpers/dr_helpers.py @@ -1715,3 +1715,49 @@ def verify_drpc_deletion(cmd, expected_output_lst): if expected_output not in drpc_out.stderr.decode(): return False return True + + +def verify_last_kubeobject_protection_time(drpc_obj, kubeobject_sync_interval): + """ + Verifies that the lastKubeObjectProtectionTime for a given DRPC object is within the expected range. + + Args: + drpc_obj (obj): DRPC object + kubeobject_sync_interval (int): The KubeObject sync interval in minutes + + Returns: + str: Current lastKubeObjectProtectionTime + + Raises: + AssertionError: If the lastKubeObjectProtectionTime is outside the expected range + (greater than or equal to two times the scheduling interval) + + """ + restore_index = config.cur_index + config.switch_acm_ctx() + last_kubeobject_protection_time = drpc_obj.get_last_kubeobject_protection_time() + if not last_kubeobject_protection_time: + assert last_kubeobject_protection_time, ( + "There is no lastKubeObjectProtectionTime. " + "Verify that certificates are included correctly in the Ramen Hub configuration map." + ) + # Verify lastGroupSyncTime + time_format = "%Y-%m-%dT%H:%M:%SZ" + last_kubeobject_protection_time_formatted = datetime.strptime( + last_kubeobject_protection_time, time_format + ) + current_time = datetime.strptime( + datetime.utcnow().strftime(time_format), time_format + ) + time_since_last_sync = ( + current_time - last_kubeobject_protection_time_formatted + ).total_seconds() / 60 + logger.info( + f"Time in minutes since the last Kube Object sync {time_since_last_sync}" + ) + assert ( + time_since_last_sync < 2 * kubeobject_sync_interval + ), "The syncing of Kube Resources is exceeding three times the Kube object sync interval" + logger.info("Verified lastKubeObjectProtectionTime value within expected range") + config.switch_ctx(restore_index) + return last_kubeobject_protection_time diff --git a/ocs_ci/ocs/dr/dr_workload.py b/ocs_ci/ocs/dr/dr_workload.py index 42162d66732..b62d758c693 100644 --- a/ocs_ci/ocs/dr/dr_workload.py +++ b/ocs_ci/ocs/dr/dr_workload.py @@ -1076,7 +1076,8 @@ def __init__(self, **kwargs): self.discovered_apps_placement_name = kwargs.get("workload_placement_name") self.drpc_yaml_file = os.path.join(constants.DRPC_PATH) self.placement_yaml_file = os.path.join(constants.PLACEMENT_PATH) - self.kubeobject_capture_interval = f"{generate_kubeobject_capture_interval()}m" + self.kubeobject_capture_interval_int = generate_kubeobject_capture_interval() + self.kubeobject_capture_interval = f"{self.kubeobject_capture_interval_int}m" self.protection_type = kwargs.get("protection_type") self.target_clone_dir = config.ENV_DATA.get( "target_clone_dir", constants.DR_WORKLOAD_REPO_BASE_DIR diff --git a/ocs_ci/ocs/resources/drpc.py b/ocs_ci/ocs/resources/drpc.py index bad71dc4d71..6972271f663 100644 --- a/ocs_ci/ocs/resources/drpc.py +++ b/ocs_ci/ocs/resources/drpc.py @@ -101,6 +101,22 @@ def get_last_group_sync_time(self): logger.info(f"Current lastGroupSyncTime is {last_group_sync_time}.") return last_group_sync_time + def get_last_kubeobject_protection_time(self): + """ + Fetch lastKubeObjectProtectionTime from DRPC + + Returns: + str: lastKubeObjectProtectionTime + + """ + last_kubeobject_protection_time = ( + self.get().get("status").get("lastKubeObjectProtectionTime") + ) + logger.info( + f"Current lastKubeObjectProtectionTime is {last_kubeobject_protection_time}." + ) + return last_kubeobject_protection_time + def get_drpc_name(namespace, switch_ctx=None): """ diff --git a/tests/functional/disaster-recovery/regional-dr/test_failover_and_relocate_discovered_apps.py b/tests/functional/disaster-recovery/regional-dr/test_failover_and_relocate_discovered_apps.py index 3cd4b4b8b25..11b805b1368 100644 --- a/tests/functional/disaster-recovery/regional-dr/test_failover_and_relocate_discovered_apps.py +++ b/tests/functional/disaster-recovery/regional-dr/test_failover_and_relocate_discovered_apps.py @@ -3,10 +3,11 @@ from ocs_ci.framework import config -from ocs_ci.framework.testlib import acceptance, tier1 +from ocs_ci.framework.testlib import acceptance, tier1, skipif_ocs_version from ocs_ci.framework.pytest_customization.marks import rdr, turquoise_squad from ocs_ci.helpers import dr_helpers - +from ocs_ci.ocs import constants +from ocs_ci.ocs.resources.drpc import DRPC logger = logging.getLogger(__name__) @@ -15,6 +16,7 @@ @acceptance @tier1 @turquoise_squad +@skipif_ocs_version("<4.16") class TestFailoverAndRelocateWithDiscoveredApps: """ Test Failover and Relocate with Discovered Apps @@ -45,10 +47,16 @@ def test_failover_and_relocate_discovered_apps(self, discovered_apps_dr_workload scheduling_interval = dr_helpers.get_scheduling_interval( rdr_workload.workload_namespace, discovered_apps=True ) + drpc_obj = DRPC(namespace=constants.DR_OPS_NAMESAPCE) wait_time = 2 * scheduling_interval # Time in minutes logger.info(f"Waiting for {wait_time} minutes to run IOs") sleep(wait_time * 60) + logger.info("Checking for lastKubeObjectProtectionTime") + dr_helpers.verify_last_kubeobject_protection_time( + drpc_obj, rdr_workload.kubeobject_capture_interval_int + ) + dr_helpers.failover( failover_cluster=secondary_cluster_name, namespace=rdr_workload.workload_namespace, @@ -87,11 +95,17 @@ def test_failover_and_relocate_discovered_apps(self, discovered_apps_dr_workload scheduling_interval = dr_helpers.get_scheduling_interval( rdr_workload.workload_namespace, discovered_apps=True ) + logger.info("Running Relocate Steps") wait_time = 2 * scheduling_interval # Time in minutes logger.info(f"Waiting for {wait_time} minutes to run IOs") sleep(wait_time * 60) + logger.info("Checking for lastKubeObjectProtectionTime") + dr_helpers.verify_last_kubeobject_protection_time( + drpc_obj, rdr_workload.kubeobject_capture_interval_int + ) + dr_helpers.relocate( preferred_cluster=secondary_cluster_name, namespace=rdr_workload.workload_namespace, @@ -101,4 +115,9 @@ def test_failover_and_relocate_discovered_apps(self, discovered_apps_dr_workload workload_instance=rdr_workload, ) + logger.info("Checking for lastKubeObjectProtectionTime post Relocate Operation") + dr_helpers.verify_last_kubeobject_protection_time( + drpc_obj, rdr_workload.kubeobject_capture_interval_int + ) + # TODO: Add data integrity checks