Merge pull request #12 from RWKV/main-dev-infctx

fixing vocab size mismatch
RWKV · Aug 21, 2023 · 505622b · 505622b
2 parents fd6bbfb + d5356bc
commit 505622b
Show file tree

Hide file tree

Showing 4 changed files with 4 additions and 4 deletions.
diff --git a/RWKV-v4neo/init_model.py b/RWKV-v4neo/init_model.py
@@ -114,7 +114,7 @@ def main():
     if vocab_size == "neox":
         vocab_size = 50277
     elif vocab_size == "world":
-        vocab_size = 65529
+        vocab_size = 65536
     else:
         vocab_size = int(vocab_size)
 

diff --git a/RWKV-v4neo/src/dataflow/trie_tokenizer.py b/RWKV-v4neo/src/dataflow/trie_tokenizer.py
@@ -52,7 +52,7 @@ def find_longest(self, key:bytes, idx:int=0):
 
 class TRIE_TOKENIZER():
     def __init__(self, file_name):
-        self.vocab_size = 65529
+        self.vocab_size = 65536
         self.idx2token = {}
         sorted = [] # must be already sorted
         with open(file_name, "r", encoding="utf-8") as f:

diff --git a/RWKV-v5/init_model.py b/RWKV-v5/init_model.py
@@ -116,7 +116,7 @@ def main():
     if vocab_size == "neox":
         vocab_size = 50277
     elif vocab_size == "world":
-        vocab_size = 65529
+        vocab_size = 65536
     else:
         vocab_size = int(vocab_size)
 

diff --git a/RWKV-v5/src/dataflow/trie_tokenizer.py b/RWKV-v5/src/dataflow/trie_tokenizer.py
@@ -52,7 +52,7 @@ def find_longest(self, key:bytes, idx:int=0):
 
 class TRIE_TOKENIZER():
     def __init__(self, file_name):
-        self.vocab_size = 65529
+        self.vocab_size = 65536
         self.idx2token = {}
         sorted = [] # must be already sorted
         with open(file_name, "r", encoding="utf-8") as f: