Merge pull request #191 from antonioramos1/fix_end_of_header_token_names

Fixes end_header_id token name in comments
meta-llama · May 14, 2024 · 14aab04 · 14aab04
2 parents f37d7d3 + b20cad1
commit 14aab04
Showing 1 changed file with 4 additions and 4 deletions.
diff --git a/llama/test_tokenizer.py b/llama/test_tokenizer.py
@@ -46,7 +46,7 @@ def test_encode_message(self):
             [
                 128006,  # <|start_header_id|>
                 882,  # "user"
-                128007,  # <|end_of_header|>
+                128007,  # <|end_header_id|>
                 271,  # "\n\n"
                 2028, 374, 264, 1296, 11914, 13,  # This is a test sentence.
                 128009,  # <|eot_id|>
@@ -70,19 +70,19 @@ def test_encode_dialog(self):
                 128000,  # <|begin_of_text|>
                 128006,  # <|start_header_id|>
                 9125,     # "system"
-                128007,  # <|end_of_header|>
+                128007,  # <|end_header_id|>
                 271,     # "\n\n"
                 2028, 374, 264, 1296, 11914, 13,  # "This is a test sentence."
                 128009,  # <|eot_id|>
                 128006,  # <|start_header_id|>
                 882,     # "user"
-                128007,  # <|end_of_header|>
+                128007,  # <|end_header_id|>
                 271,     # "\n\n"
                 2028, 374, 264, 2077, 13,  # "This is a response.",
                 128009,  # <|eot_id|>
                 128006,  # <|start_header_id|>
                 78191,   # "assistant"
-                128007,  # <|end_of_header|>
+                128007,  # <|end_header_id|>
                 271,     # "\n\n"
             ]
         )