ipu: Check if insert fails and retry

Signed-off-by: Salvatore Daniele <[email protected]>

ipu.py

@@ -354,7 +354,7 @@ def boot_iso_with_redfish(self, iso_path: str) -> None:
         logger.info("Checking if iso needs to be cleaned up")
         self.cleanup_iso_if_needed(iso_path)
         logger.info("inserting iso")
-        self._insert_media(iso_path)
+        self._insert_media_retry(iso_path)
         logger.info("setting boot source override")
         self._bootsource_override_cd()
         logger.info("triggering reboot")
@@ -376,6 +376,8 @@ def _wait_iso_downloaded(self, iso_path: str) -> None:
         rh.ssh_connect("root", password="", discover_auth=False)
         loop_count = 0
         while True:
+            if not rh.is_connected():
+                logger.error_and_exit(f"Connection to {rh.hostname} has been dropped, is download failed")
             result = rh.run("du -b /mnt/imc/acc-os.iso").out.split()
             if len(result) == 0:
                 continue
@@ -402,6 +404,19 @@ def _insert_media(self, iso_path: str) -> None:
         logger.info("Waiting for the size of iso_path to be the same the IMC")
         self._wait_iso_downloaded(iso_path)
 
+    def _insert_media_retry(self, iso_path:str, max_retries = 10) -> None:
+        retries = 0
+        while True:
+            try:
+                self._insert_media(iso_path)
+                break
+            except Exception as e:
+                logger.info(f"Encountered exception {e} when inserting media, retrying... (attempt={retries})")
+                retries += 1
+                if retries >= max_retries:
+                    logger.error_and_exit("Insert media failed")
+                time.sleep(5)
+
     def _bootsource_override_cd(self) -> None:
         url = f"https://{self.url}:8443/redfish/v1/Systems/1"
         data = {"Boot": {"BootSourceOverrideEnabled": "Once", "BootSourceOverrideTarget": "Cd"}}