Pin the version of tokenizers to compile it on macOS Python3.6

hankcs · Nov 28, 2023 · ff95dee · ff95dee
1 parent 6bbf1ff
commit ff95dee
Showing 1 changed file with 5 additions and 0 deletions.
diff --git a/setup.py b/setup.py
@@ -18,6 +18,10 @@
 else:
     TENSORFLOW = ['tensorflow==2.6.0', 'keras==2.6.0', 'protobuf<3.19']
 
+TOKENIZERS = []
+if sys.version_info == (3, 6):
+    TOKENIZERS = ['tokenizers==0.11.6']
+
 extras_require = {
     'amr': [
         'penman==1.2.1',
@@ -66,6 +70,7 @@
         'hanlp-common>=0.0.19',
         'hanlp-trie>=0.0.4',
         'hanlp-downloader',
+        *TOKENIZERS,
     ],
     extras_require=extras_require,
     python_requires='>=3.6',