smiles tokenizer

prescient-design · Jan 27, 2025 · 3921d30 · 3921d30
1 parent 11529cd
commit 3921d30
Show file tree

Hide file tree

Showing 10 changed files with 1,483 additions and 20 deletions.
diff --git a/src/lobster/assets/smiles_tokenizer/special_tokens_map.json b/src/lobster/assets/smiles_tokenizer/special_tokens_map.json
@@ -0,0 +1,6 @@
+{
+  "eos_token": "<eos>",
+  "pad_token": "<pad>",
+  "sep_token": "<sep>",
+  "unk_token": "<unk>"
+}