From 270ef771c60f5e5487a4a83e35d4d09dbdc46637 Mon Sep 17 00:00:00 2001 From: zanderjiang Date: Thu, 5 Dec 2024 05:58:19 +0000 Subject: [PATCH] update testcase for tokenizer_info --- tests/python/test_tokenizer_info.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python/test_tokenizer_info.py b/tests/python/test_tokenizer_info.py index 9927f8e..ced8090 100644 --- a/tests/python/test_tokenizer_info.py +++ b/tests/python/test_tokenizer_info.py @@ -41,7 +41,7 @@ def tokenizer_info_storage() -> Dict[str, Tuple[PreTrainedTokenizerBase, xgr.Tok ("Qwen/Qwen2.5-1.5B", xgr.VocabType.BYTE_LEVEL, False), ("internlm/internlm2_5-7b-chat", xgr.VocabType.BYTE_FALLBACK, False), ("mistralai/Mixtral-8x22B-Instruct-v0.1", xgr.VocabType.BYTE_FALLBACK, True), - ("THUDM/LongWriter-glm4-9b", xgr.VocabType.RAW, False), + ("THUDM/glm-4-9b-chat", xgr.VocabType.RAW, False), ("THUDM/chatglm3-6b", xgr.VocabType.BYTE_FALLBACK, True), ]