feat: Norvig's spelling corrector loaded and tested

src/lua/init.lua

@@ -67,6 +67,7 @@ TIME = require'time'
 INSPECT = require('inspect')
 QSORT = require('qsort_op') -- optimized table sort
 table.sort = QSORT -- override native table sort
+SPELL = require('spell')
 JSON = require('zenroom_json')
 ECDH = require('zenroom_ecdh')
 -- ECDH public keys cannot function as ECP because of IANA 7303

src/lua/spell.lua

@@ -0,0 +1,105 @@
+-- Spelling Corrector.
+--
+-- Copyright 2014 Francisco Zamora-Martinez
+-- Copyright 2024 Jaromil (Dyne.org)
+-- Adaptation of Peter Norvig python Spelling Corrector:
+-- http://norvig.com/spell-correct.html
+-- Open source code under MIT license: http://www.opensource.org/licenses/mit-license.php
+
+local yield,wrap = coroutine.yield,coroutine.wrap
+local alphabet_str,alphabet = 'abcdefghijklmnopqrstuvwxyz',{}
+for a in alphabet_str:gmatch(".") do alphabet[#alphabet+1] = a end
+spell = {}
+
+local function list(w) return pairs{[w]=true} end
+
+function spell:max(...)
+  local arg,max,hyp = table.pack(...),0,nil
+  for w in table.unpack(arg) do
+    local p = self.model[w] or 1
+	if p>max or ( p==max and hyp<w ) then hyp,max=w,p end
+  end
+  return hyp
+end
+
+-- local function words(text) return text:lower():gmatch("[a-z]+") end
+
+-- local function train(features)
+--   for f in features do model[f] = (model[f] or 1) + 1 end
+-- end
+
+-- local function init(filename) train(words(io.open(filename):read("*a"))) end
+
+local function make_yield()
+  local set = {}
+  return function(w)
+	 if not set[w] then
+		set[w] = true
+		yield(w)
+	 end
+  end
+end
+
+local function edits1(word_str, yield)
+   local yield = yield or make_yield()
+   return wrap(function()
+		 local splits, word = {}, {}
+		 for i=1,#word_str do
+			word[i],splits[i] = word_str:sub(i,i),{word_str:sub(1,i),word_str:sub(i)}
+		 end
+		 -- sentinels
+		 splits[0], splits[#word_str+1] = { "", word_str }, { word_str, ""}
+		 -- deletes
+		 for i=1,#word_str do yield( splits[i-1][1]..splits[i+1][2] ) end
+		 -- transposes
+		 for i=1,#word_str-1 do
+			yield( splits[i-1][1]..word[i+1]..word[i]..splits[i+2][2] )
+		 end
+		 -- replaces
+		 for i=1,#word_str do
+			for j=1,#alphabet do
+			   yield( splits[i-1][1]..alphabet[j]..splits[i+1][2] )
+			end
+		 end
+		 -- inserts
+		 for i=0,#word_str do
+			for j=1,#alphabet do
+			   yield( splits[i][1]..alphabet[j]..splits[i+1][2] )
+			end
+		 end
+   end)
+end
+
+function spell:known_edits2(w, set)
+  local yield,yield2 = make_yield(),make_yield()
+  return wrap(function()
+      for e1 in edits1(w) do
+		 for e2 in edits1(e1,yield2) do
+			if self.model[e2] then yield( e2 ) end
+		 end
+	  end
+  end)
+end
+
+function spell:known(list,aux)
+  return wrap(function()
+      for w in list,aux do
+		 if self.model[w] then yield(w) end
+	  end
+  end)
+end
+
+function spell:correct(w)
+  local w = w:lower()
+  local result = self:max(self:known(list(w)))
+	 or self:max(self:known(edits1(w)))
+	 or self:max(self:known_edits2(w))
+	 or self:max(list(w))
+  if result then
+	 return result
+  else
+	 return false,"No suggestion found for word: "..w
+  end
+end
+
+return spell

test/lua/introspection.lua

@@ -74,3 +74,9 @@ print''
 print( 'Total Zencode statements: '..#statements)
 print( 'Total unique word tokens: '..table_size(tokens))
 print''
+
+SPELL.model = tokens
+assert( SPELL:correct('crate') == 'create' )
+assert( SPELL:correct('cruute') == 'create' )
+assert( SPELL:correct('frr') == 'for' )
+assert( SPELL:correct('uerfy') == 'verify' )