package-indexer: always calculate md5 locally

Signed-off-by: Attila Szakacs <[email protected]>

packaging/package-indexer/remote_storage_synchronizer/azure_container_synchronizer.py

@@ -20,7 +20,6 @@
 #
 #############################################################################
 
-from hashlib import md5
 from pathlib import Path
 from typing import Any, Dict, List
 
@@ -100,12 +99,6 @@ def _create_snapshot_of_remote(self) -> None:
                 snapshot_properties=str(snapshot_properties),
             )
 
-    def _get_md5_of_remote_file(self, relative_file_path: str) -> bytes:
-        for file in self._remote_files:
-            if file["name"] == relative_file_path:
-                return file["content_settings"]["content_md5"]
-        raise FileNotFoundError
-
     def _get_relative_file_path_for_remote_file(self, file: Dict[str, Any]) -> str:
         return file["name"]
 

packaging/package-indexer/remote_storage_synchronizer/remote_storage_synchronizer.py

@@ -172,6 +172,11 @@ def __download_file(self, relative_file_path: str) -> None:
         self._download_file(relative_file_path)
         self._log_info("Successfully downloaded file.", remote_path=relative_file_path, local_path=str(download_path))
 
+        md5sum = md5(download_path.read_bytes()).digest()
+        md5sum_file_path = self.__get_remote_md5sum_file_path(relative_file_path)
+        md5sum_file_path.parent.mkdir(exist_ok=True, parents=True)
+        md5sum_file_path.write_bytes(md5sum)
+
     @abstractmethod
     def _upload_file(self, relative_file_path: str) -> None:
         pass
@@ -212,9 +217,13 @@ def __get_md5_of_local_file(self, relative_file_path: str) -> bytes:
         file = Path(self.local_dir.root_dir, relative_file_path)
         return md5(file.read_bytes()).digest()
 
-    @abstractmethod
-    def _get_md5_of_remote_file(self, relative_file_path: str) -> bytes:
-        pass
+    def __get_remote_md5sum_file_path(self, relative_file_path: str) -> Path:
+        path = Path(self.local_dir.root_dir, "package-indexer-md5sums", relative_file_path).resolve()
+        return Path(path.parent, path.name + ".package-indexer-md5sum")
+
+    def __get_md5_of_remote_file(self, relative_file_path: str) -> bytes:
+        md5sum_file_path = self.__get_remote_md5sum_file_path(relative_file_path)
+        return md5sum_file_path.read_bytes()
 
     def __get_file_sync_state(self, relative_file_path: str) -> FileSyncState:
         try:
@@ -228,7 +237,7 @@ def __get_file_sync_state(self, relative_file_path: str) -> FileSyncState:
             return FileSyncState.NOT_IN_LOCAL
 
         try:
-            remote_md5 = self._get_md5_of_remote_file(relative_file_path)
+            remote_md5 = self.__get_md5_of_remote_file(relative_file_path)
         except FileNotFoundError:
             self._log_debug(
                 "Local file is not available remotely.",

packaging/package-indexer/remote_storage_synchronizer/s3_bucket_synchronizer.py

@@ -20,7 +20,6 @@
 #
 #############################################################################
 
-from hashlib import md5
 from pathlib import Path
 from typing import Any, Dict, List
 
@@ -116,21 +115,12 @@ def _list_remote_files(self) -> List[Dict[str, Any]]:
 
         return objects
 
-    def __get_remote_md5sum_file_path(self, relative_file_path: str) -> Path:
-        path = Path(self.local_dir.root_dir, "package-indexer-md5sums", relative_file_path).resolve()
-        return Path(path.parent, path.name + ".package-indexer-md5sum")
-
     def _download_file(self, relative_file_path: str) -> None:
         download_path = self._get_local_file_path_for_relative_file(relative_file_path)
         download_path.parent.mkdir(parents=True, exist_ok=True)
         with download_path.open("wb") as downloaded_object:
             self.__client.download_fileobj(self.__bucket, relative_file_path, downloaded_object)
 
-        md5sum = md5(download_path.read_bytes()).digest()
-        md5sum_file_path = self.__get_remote_md5sum_file_path(relative_file_path)
-        md5sum_file_path.parent.mkdir(exist_ok=True, parents=True)
-        md5sum_file_path.write_bytes(md5sum)
-
     def _upload_file(self, relative_file_path: str) -> None:
         local_path = self._get_local_file_path_for_relative_file(relative_file_path)
         with local_path.open("rb") as local_file_data:
@@ -142,10 +132,6 @@ def _delete_remote_file(self, relative_file_path: str) -> None:
     def _create_snapshot_of_remote(self) -> None:
         self._log_info("Cannot create snapshot, not implemented, skipping...")
 
-    def _get_md5_of_remote_file(self, relative_file_path: str) -> bytes:
-        md5sum_file_path = self.__get_remote_md5sum_file_path(relative_file_path)
-        return md5sum_file_path.read_bytes()
-
     def _get_relative_file_path_for_remote_file(self, file: Dict[str, Any]) -> str:
         return file["Key"]