Modify package structure, start using Click, add phon_fst

MANIFEST.in

@@ -0,0 +1 @@
+include timur/data/syms.txt

requirements.txt

@@ -1 +1,2 @@
+click
 http://www.opengrm.org/twiki/pub/GRM/PyniniDownload/pynini-2.0.0.tar.gz#egg=pynini

setup.py

@@ -17,6 +17,7 @@
     author_email='[email protected]',
     license=license,
     packages=find_packages(exclude=('tests', 'docs')),
+    include_package_data=True,
     install_requires=[
     ],
     entry_points={

timur/fsts/__init__.py

@@ -1 +1,2 @@
 from .num_fst import num_fst
+from .phon_fst import phon_fst

timur/fsts/phon_fst.py

@@ -0,0 +1,15 @@
+# -*- coding: utf-8 -*- 
+from __future__ import absolute_import
+
+import pynini
+
+from timur.helpers import union
+from timur.helpers import concat
+
+def phon_fst(symbol_table):
+    '''
+    Orthographic and phonological surface realizations rules
+    '''
+    cons_lower = pynini.string_map(["b", "c", "d", "f", "g", "h", "j", "k", "l", "m", "n", "p", "q", "r", "s", "t", "v", "w", "x", "y", "z", "ß"], input_token_type=symbol_table, output_token_type=symbol_table)
+    cons_upper = pynini.string_map(["B", "C", "D", "F", "G", "H", "J", "K", "L", "M", "N", "P", "Q", "R", "S", "T", "V", "W", "X", "Y", "Z"], input_token_type=symbol_table, output_token_type=symbol_table)
+    #return cons.optimize()

timur/helpers/helpers.py

@@ -29,7 +29,7 @@ def load_alphabet(source, auto_singletons=True):
             symbol = chr(i)
             if symbol.isprintable() and not symbol.isspace():
                 syms.add_symbol(symbol)
-    for symbol in source:
+    for symbol in source.split('\n'):
         if symbol.startswith('#'):
             continue
         syms.add_symbol(symbol.strip())

timur/scripts/timur.py

@@ -1,7 +1,9 @@
 # -*- coding: utf-8 -*- 
 from __future__ import absolute_import
 
-import pynini
+import click, pynini
+
+from pkg_resources import resource_string, Requirement
 
 from timur import helpers
 from timur import fsts
@@ -20,22 +22,21 @@ def construct_any(symbol_table):
         sym_it.next()
     return ANY
 
-def phon_fst(symbol_table):
-    '''
-    Orthographic and phonological surface realizations rules
-    '''
-    cons_lower = pynini.string_map(["b", "c", "d", "f", "g", "h", "j", "k", "l", "m", "n", "p", "q", "r", "s", "t", "v", "w", "x", "y", "z", "ß"], input_token_type=symbol_table, output_token_type=symbol_table)
-    cons_upper = pynini.string_map(["B", "C", "D", "F", "G", "H", "J", "K", "L", "M", "N", "P", "Q", "R", "S", "T", "V", "W", "X", "Y", "Z"], input_token_type=symbol_table, output_token_type=symbol_table)
-    #return cons.optimize()
+@click.group()
+def cli():
+    pass
 
 
-def cli():
-    syms = helpers.load_alphabet(open("syms.txt"))
+@cli.command(name="compile")
+@click.argument('lexicon')
+def compile(lexicon):
+
+    syms = helpers.load_alphabet(resource_string(Requirement.parse("timur"), 'timur/data/syms.txt').decode("utf-8"))
 
     #phon = phon_fst(syms)
     #phon.draw("test.dot")
     num_stems = fsts.num_fst(syms)
 
     ANY = construct_any(syms)
 
-    print(syms.member('A'))
+    print(syms.member('<QUANT>'))