checked for conflicts in file hashes

mit-submit · Oct 13, 2023 · 2a65ff0 · 2a65ff0
1 parent 623c229
commit 2a65ff0
Showing 1 changed file with 3 additions and 0 deletions.
diff --git a/A2rchi/utils/data_manager.py b/A2rchi/utils/data_manager.py
@@ -225,6 +225,9 @@ def _add_to_vectorstore(self, collection, files_to_add, sources={}):
                 time_identifier = hashlib.md5()
                 time_identifier.update(str(time.time()).encode('utf-8'))
                 time_hash = str(int(identifier.hexdigest(),16))[0:6]
+                while str(filehash) + str(chunk_hash) + str(time_hash) in ids:
+                    print("INFO: Found conflict with hash: " + str(filehash) + str(chunk_hash) + str(time_hash) + ". Trying again")
+                    time_hash += 1
                 ids.append(str(filehash) + str(chunk_hash) + str(time_hash))
 
             print("Ids: ",ids)