s/sentencepiece/tiktoken

ruanslv · ruanslv · commit 3afbe13a9b7b · 2024-04-14T15:58:15.000-07:00
diff --git a/llama/tokenizer.py b/llama/tokenizer.py
@@ -82,7 +82,7 @@ def __init__(self, model_path: str):
             mergeable_ranks=mergeable_ranks,
             special_tokens=self.special_tokens,
         )
-        logger.info(f"Reloaded SentencePiece model from {model_path}")
+        logger.info(f"Reloaded tiktoken model from {model_path}")
 
         # BOS / EOS token IDs
         self.n_words: int = self.model.n_vocab

Original file line number	Diff line number	Diff line change
`@@ -82,7 +82,7 @@ def __init__(self, model_path: str):`
`82`	`82`	`mergeable_ranks=mergeable_ranks,`
`83`	`83`	`special_tokens=self.special_tokens,`
`84`	`84`	`)`
`85`		`- logger.info(f"Reloaded SentencePiece model from {model_path}")`
	`85`	`+ logger.info(f"Reloaded tiktoken model from {model_path}")`
`86`	`86`
`87`	`87`	`# BOS / EOS token IDs`
`88`	`88`	`self.n_words: int = self.model.n_vocab`