Develop lx

src/tokenizers/Unigram/trie.hpp

@@ -68,11 +68,11 @@ class TrieIterator {
                 node = next->second;
                 path.push_back(*iter);
                 if (node->is_leaf) {
-                    iter++;
+                    ++iter;
                     return path;
                 }
             }
-            iter++;
+            ++iter;
         }
         return {};
 

test/preprocess/UnigramTest.cpp

@@ -10,10 +10,19 @@ TEST_F(TokenizerTest, test) {
     tokenizer->setSpecialToken("|ENDOFTEXT|");
     std::string text = "Hello world";
      // normalization text
-    // replace all " " to "_"
-    std::replace(text.begin(), text.end(), ' ', '_');
+    // replace all " " to "▁"
+    std::string text_ = "";
+    for (auto &ch : text) {
+        if (ch == ' ') {
+            text_ += "▁";
+        }else {
+            text_ += ch;
+        }
+
+    }
+    // std::replace(text.begin(), text.end(), ' ', L'▁');
     // prepend "_" to text
-    std::string new_text = "_" + std::string(text);
+    std::string new_text = "▁" + std::string(text_);
 
     tokenizer->tokenize(new_text, ids, true);
     for (auto id : ids) {

test/preprocess/fuyu_process_test.py

@@ -4,10 +4,10 @@
 
 model_id = "adept/fuyu-8b"
 processor = FuyuProcessor.from_pretrained(model_id)
-text_prompt = "Generate a coco-style caption.\n"
+text_prompt = "Hello world"
 url = "https://huggingface.co/adept/fuyu-8b/resolve/main/bus.png"
-image = Image.open(requests.get(url, stream=True).raw)
-with open("bus.tmp", "wb") as f:
-    f.write(image.tobytes())
-inputs = processor(text=text_prompt, images=image, return_tensors="pt")
+# image = Image.open(requests.get(url, stream=True).raw)
+# with open("bus.tmp", "wb") as f:
+#     f.write(image.tobytes())
+inputs = processor(text=text_prompt, return_tensors="pt")
 print(inputs)