diff --git a/ipu.py b/ipu.py index 02547c11..cb5fbc6f 100644 --- a/ipu.py +++ b/ipu.py @@ -354,7 +354,7 @@ def boot_iso_with_redfish(self, iso_path: str) -> None: logger.info("Checking if iso needs to be cleaned up") self.cleanup_iso_if_needed(iso_path) logger.info("inserting iso") - self._insert_media(iso_path) + self._insert_media_retry(iso_path) logger.info("setting boot source override") self._bootsource_override_cd() logger.info("triggering reboot") @@ -376,6 +376,8 @@ def _wait_iso_downloaded(self, iso_path: str) -> None: rh.ssh_connect("root", password="", discover_auth=False) loop_count = 0 while True: + if not rh.is_connected(): + logger.error_and_exit(f"Connection to {rh.hostname} has been dropped, is download failed") result = rh.run("du -b /mnt/imc/acc-os.iso").out.split() if len(result) == 0: continue @@ -402,6 +404,19 @@ def _insert_media(self, iso_path: str) -> None: logger.info("Waiting for the size of iso_path to be the same the IMC") self._wait_iso_downloaded(iso_path) + def _insert_media_retry(self, iso_path:str, max_retries = 10) -> None: + retries = 0 + while True: + try: + self._insert_media(iso_path) + break + except Exception as e: + logger.info(f"Encountered exception {e} when inserting media, retrying... (attempt={retries})") + retries += 1 + if retries >= max_retries: + logger.error_and_exit("Insert media failed") + time.sleep(5) + def _bootsource_override_cd(self) -> None: url = f"https://{self.url}:8443/redfish/v1/Systems/1" data = {"Boot": {"BootSourceOverrideEnabled": "Once", "BootSourceOverrideTarget": "Cd"}}