Put a levenshtein max distance check into the phonetic matcher.

Author: pudo

nomenklatura/matching/compare/names.py

@@ -4,23 +4,18 @@
 from followthemoney.types import registry
 from fingerprints import clean_name_light, clean_name_ascii
 from rigour.text.distance import levenshtein_similarity
-from rigour.text.distance import dam_levenshtein, jaro_winkler
+from rigour.text.distance import jaro_winkler
 from nomenklatura.util import names_word_list, name_words
 from nomenklatura.util import fingerprint_name, normalize_name
 from nomenklatura.matching.util import type_pair, props_pair, has_schema
 from nomenklatura.matching.compare.util import is_disjoint, clean_map, has_overlap
+from nomenklatura.matching.compare.util import is_levenshtein_plausible
 
 
 def _name_parts(name: str) -> List[str]:
     return name_words(normalize_name(name))
 
 
-def _is_levenshtein_plausible(query: str, result: str) -> bool:
-    # Skip results with an overall distance of more than 3 characters:
-    max_edits = min(3, (min(len(query), len(result)) // 3))
-    return dam_levenshtein(query, result) <= max_edits
-
-
 def _align_name_parts(query: List[str], result: List[str]) -> float:
     if len(query) == 0 or len(result) == 0:
         return 0.0
@@ -29,7 +24,7 @@ def _align_name_parts(query: List[str], result: List[str]) -> float:
     # compute all pairwise scores for name parts:
     for qn, rn in product(set(query), set(result)):
         score = jaro_winkler(qn, rn)
-        if score > 0.0 and _is_levenshtein_plausible(qn, rn):
+        if score > 0.0 and is_levenshtein_plausible(qn, rn):
             scores[(qn, rn)] = score
     pairs: List[Tuple[str, str]] = []
     # original length of query:
@@ -50,7 +45,7 @@ def _align_name_parts(query: List[str], result: List[str]) -> float:
     aligned = pairs[::-1]
     query_aligned = "".join(p[0] for p in aligned)
     result_aligned = "".join(p[1] for p in aligned)
-    if not _is_levenshtein_plausible(query_aligned, result_aligned):
+    if not is_levenshtein_plausible(query_aligned, result_aligned):
         return 0.0
     # return an amped-up jaro-winkler score for the aligned name parts:
     return total_score
@@ -68,7 +63,7 @@ def person_name_jaro_winkler(query: E, result: E) -> float:
     for (qn, rn) in product(query_names, result_names):
         qns = "".join(qn)
         rns = "".join(rn)
-        if _is_levenshtein_plausible(qns, rns):
+        if is_levenshtein_plausible(qns, rns):
             score = max(score, jaro_winkler(qns, rns) ** len(qns))
         score = max(score, _align_name_parts(list(qn), list(rn)))
     return score

nomenklatura/matching/compare/phonetic.py

@@ -3,18 +3,33 @@
 from followthemoney.proxy import E
 from followthemoney.types import registry
 from rigour.text.scripts import is_modern_alphabet
-from fingerprints import clean_name_ascii, clean_entity_prefix
+from rigour.names.part import name_parts, NamePart
 from nomenklatura.util import name_words, list_intersection, fingerprint_name
-from nomenklatura.util import phonetic_token, metaphone_token, soundex_token
+from nomenklatura.util import metaphone_token, soundex_token
 from nomenklatura.matching.util import type_pair, has_schema
+from nomenklatura.matching.compare.util import is_levenshtein_plausible
 
 
-def _clean_phonetic_person(original: str) -> Optional[str]:
-    """Normalize a person name without transliteration."""
-    if not is_modern_alphabet(original):
-        return None
-    text = clean_entity_prefix(original)
-    return clean_name_ascii(text)
+def compare_parts_phonetic(left: NamePart, right: NamePart) -> bool:
+    if left.metaphone is None or right.metaphone is None:
+        return left.ascii == right.ascii
+    if (
+        left.metaphone == right.metaphone
+        and left.ascii is not None
+        and right.ascii is not None
+    ):
+        # Secondary check for Levenshtein distance:
+        if is_levenshtein_plausible(left.ascii, right.ascii):
+            return True
+    return False
+
+
+# def _clean_phonetic_person(original: str) -> Optional[str]:
+#     """Normalize a person name without transliteration."""
+#     if not is_modern_alphabet(original):
+#         return None
+#     text = clean_entity_prefix(original)
+#     return clean_name_ascii(text)
 
 
 def _clean_phonetic_entity(original: str) -> Optional[str]:
@@ -24,11 +39,11 @@ def _clean_phonetic_entity(original: str) -> Optional[str]:
     return fingerprint_name(original)
 
 
-def _phonetic_person_tokens(token: str) -> List[str]:
-    words: List[str] = []
-    for word in name_words(_clean_phonetic_person(token), min_length=2):
-        words.append(phonetic_token(word))
-    return words
+# def _phonetic_person_tokens(token: str) -> List[str]:
+#     words: List[str] = []
+#     for word in name_words(_clean_phonetic_person(token), min_length=2):
+#         words.append(phonetic_token(word))
+#     return words
 
 
 def _token_names_compare(
@@ -48,9 +63,22 @@ def person_name_phonetic_match(query: E, result: E) -> float:
     if not has_schema(query, result, "Person"):
         return 0.0
     query_names_, result_names_ = type_pair(query, result, registry.name)
-    query_names = [_phonetic_person_tokens(n) for n in query_names_]
-    result_names = [_phonetic_person_tokens(n) for n in result_names_]
-    return _token_names_compare(query_names, result_names)
+    query_parts = [name_parts(n) for n in query_names_]
+    result_parts = [name_parts(n) for n in result_names_]
+    score = 0.0
+    for (q, r) in product(query_parts, result_parts):
+        if len(q) == 0:
+            continue
+        matches = list(r)
+        matched = 0
+        for part in q:
+            for other in matches:
+                if compare_parts_phonetic(part, other):
+                    matches.remove(other)
+                    matched += 1
+                    break
+        score = max(score, matched / float(len(q)))
+    return score
 
 
 def _metaphone_tokens(token: str) -> List[str]:

nomenklatura/matching/compare/util.py

@@ -1,5 +1,6 @@
 import re
 from typing import List, Set, Union, Iterable, Callable, Optional
+from rigour.text.distance import dam_levenshtein
 
 CleanFunc = Optional[Callable[[str], Optional[str]]]
 FIND_NUM = re.compile(r"\d{1,}")
@@ -49,3 +50,12 @@ def extract_numbers(values: List[str]) -> Set[str]:
     for value in values:
         numbers.update(FIND_NUM.findall(value))
     return numbers
+
+
+def is_levenshtein_plausible(query: str, result: str) -> bool:
+    """A sanity check to post-filter name matching results based on a budget
+    of allowed Levenshtein distance. This basically cuts off results where
+    the Jaro-Winkler or Metaphone comparison was too lenient."""
+    # Skip results with an overall distance of more than 3 characters:
+    max_edits = min(3, (min(len(query), len(result)) // 3))
+    return dam_levenshtein(query, result) <= max_edits

nomenklatura/util.py

@@ -4,9 +4,9 @@
 from datetime import datetime, timezone
 from followthemoney import model
 from functools import lru_cache, cache
-from jellyfish import metaphone, soundex
 from normality import collapse_spaces, category_replace
 from normality.constants import WS
+from rigour.text import metaphone, soundex
 from collections.abc import Mapping, Sequence
 from fingerprints.cleanup import clean_name_ascii, clean_entity_prefix
 from fingerprints.cleanup import CHARACTERS_REMOVE_RE
@@ -150,7 +150,6 @@ def phonetic_token(token: str) -> str:
     return metaphone_token(token)
 
 
-@lru_cache(maxsize=1024)
 def metaphone_token(token: str) -> str:
     if token.isalpha() and len(token) > 1:
         out = metaphone(token)
@@ -160,7 +159,6 @@ def metaphone_token(token: str) -> str:
     return token.upper()
 
 
-@lru_cache(maxsize=1024)
 def soundex_token(token: str) -> str:
     if token.isalpha() and len(token) > 1:
         out = soundex(token)

tests/matching/test_names.py

@@ -82,14 +82,14 @@ def test_duplicative_name_similarity():
 def test_single_name():
     name = e("Person", name="Hannibal")
     other = e("Person", name="Hannibal")
-    assert person_name_phonetic_match(name, other) == 0.5
+    assert person_name_phonetic_match(name, other) == 1.0
     assert person_name_jaro_winkler(name, other) == 1.0
 
     other = e("Person", name="Hanniball")
-    assert person_name_phonetic_match(name, other) == 0.5
+    assert person_name_phonetic_match(name, other) == 1.0
 
     other = e("Person", name="Hannibol")
-    assert person_name_phonetic_match(name, other) == 0.5
+    assert person_name_phonetic_match(name, other) == 1.0
     assert person_name_jaro_winkler(name, other) > 0.8
     assert person_name_jaro_winkler(name, other) < 1.0
 
@@ -144,6 +144,14 @@ def test_person_name_phonetic_match():
     result = e("Person", name="Фуад Гулієв")
     assert person_name_phonetic_match(query, result) < 1.0
 
+    query = e("Person", name="Olga Barynova")
+    result = e("Person", name="Oleg BARANOV")
+    assert person_name_phonetic_match(query, result) < 0.6
+
+    query = e("Person", name="Ginta Boreza")
+    result = e("Person", name="Janett Borez")
+    assert person_name_phonetic_match(query, result) < 0.6
+
     query = e("Person", name="Shaikh Isa Bin Tarif Al Bin Ali")
     result = e("Person", name="Shaikh Isa Bin Tarif Al Bin Ali")
     assert person_name_phonetic_match(query, result) == 1.0
@@ -264,8 +272,8 @@ def test_jaro_lindemann():
 def test_name_alphabets():
     query = e("Person", name="Ротенберг Аркадий")
     result = e("Person", name="Arkadiii Romanovich Rotenberg")
-    assert person_name_phonetic_match(query, result) > 0.0
-    assert person_name_phonetic_match(query, result) < 0.7
+    # assert person_name_phonetic_match(query, result) > 0.0
+    assert person_name_phonetic_match(query, result) > 0.7
     assert person_name_jaro_winkler(query, result) > 0.7
 
     query = e("Person", name="Osama bin Laden")