Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add Unicode and Windows support #20

Open
wants to merge 6 commits into
base: master
Choose a base branch
from
Open

Add Unicode and Windows support #20

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
6272927
Add unicode character support for levenshtein and osa
LEFTazs Feb 3, 2020
9d64fb8
Add Windows and unicode support for damerau
LEFTazs Feb 3, 2020
58a00ef
Remove length arguments from the C methods
LEFTazs Feb 4, 2020
530042b
Fix negative indexing error
LEFTazs Feb 5, 2020
c0d9b33
Add legacy code for backwards compatibility
LEFTazs Feb 6, 2020
454af44
Refractor legacy code
LEFTazs Feb 7, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
58 changes: 58 additions & 0 deletions test/test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -183,3 +183,61 @@ def test_dl(self):
self.assertEqual(dam_lev('bca', 'ab'), 2)
self.assertEqual(dam_lev('ab', 'bdca'), 3)
self.assertEqual(dam_lev('bdca', 'ab'), 3)


class TestClevWithUnicode(unittest.TestCase):

def setUp(self):
self.iw = np.ones(10001, dtype=np.float64)
self.dw = np.ones(10001, dtype=np.float64)
self.sw = np.ones((10001, 10001), dtype=np.float64)
self.tw = np.ones((10001, 10001), dtype=np.float64)
self.iw[ord("á")] = 2.0
self.dw[ord("á")] = 2.0
self.iw[ord("ő")] = 9.0
self.dw[ord("ő")] = 9.0
self.iw[ord("Ұ")] = 10.0
self.dw[ord("Ұ")] = 10.0


def _lev(self, x, y):
return lev(x, y, self.iw, self.dw, self.sw)

def _osa(self, x, y):
return osa(x, y, self.iw, self.dw, self.sw, self.tw)

def _dl(self, x, y):
return dam_lev(x, y, self.iw, self.dw, self.sw, self.tw)

def test_lev(self):
try:
self.assertEqual(self._lev('átívelődök', 'átívelődök'), 0.0)
self.assertEqual(self._lev('', 'átívelődök'), 19.0)
self.assertEqual(self._lev('átívelődök', ''), 19.0)
self.assertEqual(self._lev('', ''), 0.0)
self.assertEqual(self._lev('átívelődök', 'átívelőd'), 2.0)
self.assertEqual(self._lev('', 'ҰǴʚΏ¤☣✐'), 16.0)
except UnicodeEncodeError:
self.fail("Could not handle special characters")

def test_osa(self):
try:
self.assertEqual(self._osa('átívelődök', 'átívelődök'), 0.0)
self.assertEqual(self._osa('', 'átívelődök'), 19.0)
self.assertEqual(self._osa('átívelődök', ''), 19.0)
self.assertEqual(self._osa('', ''), 0.0)
self.assertEqual(self._osa('átívelődök', 'átívelőd'), 2.0)
self.assertEqual(self._osa('', 'ҰǴʚΏ¤☣✐'), 16.0)
except UnicodeEncodeError:
self.fail("Could not handle special characters")

def test_dl(self):
try:
self.assertEqual(self._dl('átívelődök', 'átívelődök'), 0.0)
self.assertEqual(self._dl('', 'átívelődök'), 19.0)
self.assertEqual(self._dl('átívelődök', ''), 19.0)
self.assertEqual(self._dl('', ''), 0.0)
self.assertEqual(self._dl('átívelődök', 'átívelőd'), 2.0)
self.assertEqual(self._dl('', 'ҰǴʚΏ¤☣✐'), 16.0)
except UnicodeEncodeError:
self.fail("Could not handle special characters")
35 changes: 33 additions & 2 deletions weighted_levenshtein/clev.pxd
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,39 @@ from libc.float cimport DBL_MAX as DTYPE_MAX
ctypedef double DTYPE_t

cdef enum:
ALPHABET_SIZE = 128
ALPHABET_SIZE = 512


cdef DTYPE_t c_damerau_levenshtein_unicode(
unsigned int* word_m,
Py_ssize_t len1,
unsigned int* word_n,
Py_ssize_t len2,
DTYPE_t[::1] insert_costs,
DTYPE_t[::1] delete_costs,
DTYPE_t[:,::1] substitute_costs,
DTYPE_t[:,::1] transpose_costs) nogil


cdef DTYPE_t c_optimal_string_alignment_unicode(
unsigned int* word_m,
Py_ssize_t len1,
unsigned int* word_n,
Py_ssize_t len2,
DTYPE_t[::1] insert_costs,
DTYPE_t[::1] delete_costs,
DTYPE_t[:,::1] substitute_costs,
DTYPE_t[:,::1] transpose_costs) nogil


cdef DTYPE_t c_levenshtein_unicode(
unsigned int* word_m,
Py_ssize_t len1,
unsigned int* word_n,
Py_ssize_t len2,
DTYPE_t[::1] insert_costs,
DTYPE_t[::1] delete_costs,
DTYPE_t[:,::1] substitute_costs) nogil


cdef DTYPE_t c_damerau_levenshtein(
Expand Down Expand Up @@ -36,4 +68,3 @@ cdef DTYPE_t c_levenshtein(
DTYPE_t[::1] insert_costs,
DTYPE_t[::1] delete_costs,
DTYPE_t[:,::1] substitute_costs) nogil

Loading