Excluded relationship properties when doing property comparison (to f…

…ind duplicate IDs)
CBIIT · Mar 8, 2022 · ae18b7d · ae18b7d
1 parent 3d9511f
commit ae18b7d
Show file tree

Hide file tree

Showing 2 changed files with 14 additions and 4 deletions.
diff --git a/bento b/bento
diff --git a/data_loader.py b/data_loader.py
@@ -18,7 +18,7 @@
 from bento.common.utils import get_logger, NODES_CREATED, RELATIONSHIP_CREATED, UUID, \
     RELATIONSHIP_TYPE, MULTIPLIER, ONE_TO_ONE, DEFAULT_MULTIPLIER, UPSERT_MODE, \
     NEW_MODE, DELETE_MODE, NODES_DELETED, RELATIONSHIP_DELETED, combined_dict_counters, \
-    MISSING_PARENT, NODE_LOADED
+    MISSING_PARENT, NODE_LOADED, get_string_md5
 
 NODE_TYPE = 'type'
 PROP_TYPE = 'Type'
@@ -127,6 +127,16 @@ def check_encoding(file_name):
         return windows1252
 
 
+# Mask all relationship properties, so they won't participate in property comparison
+def get_props_signature(props):
+    clean_props = props
+    for key in clean_props.keys():
+        if '$' in key:
+            clean_props[key] = ''
+    signature = get_string_md5(str(clean_props))
+    return signature
+
+
 class DataLoader:
     def __init__(self, driver, schema, plugins=None):
         if plugins is None:
@@ -489,7 +499,7 @@ def validate_file(self, file_name, max_violations):
                 node_id = self.schema.get_id(obj)
                 if node_id:
                     if node_id in ids:
-                        if props != ids[node_id]['props']:
+                        if get_props_signature(props) != ids[node_id]['props']:
                             validation_failed = True
                             self.log.error(
                                 f'Invalid data at line {line_num}: duplicate {id_field}: {node_id}, found in line: '
@@ -502,7 +512,7 @@ def validate_file(self, file_name, max_violations):
                                 f'Duplicated data at line {line_num}: duplicate {id_field}: {node_id}, found in line: '
                                 f'{", ".join(ids[node_id]["lines"])}')
                     else:
-                        ids[node_id] = {'props': props, 'lines': [str(line_num)]}
+                        ids[node_id] = {'props': get_props_signature(props), 'lines': [str(line_num)]}
 
                 validate_result = self.schema.validate_node(obj[NODE_TYPE], obj)
                 if not validate_result['result']: