Use tiktoken

requirements.txt

@@ -1,4 +1,5 @@
 grpcio==1.62.1
 protobuf==5.26.1
 https://github.com/riccardobl/openagents-grpc-proto/releases/download/v0.6/openagents_grpc_proto-PYTHON.tar.gz
-sentence-transformers==2.6.1
+sentence-transformers==2.6.1
+tiktoken

src/main.py

@@ -13,7 +13,7 @@
 import traceback
 import base64
 from sentence_transformers.quantization import quantize_embeddings
-
+import tiktoken
 
 
 class Runner (JobRunner):
@@ -30,12 +30,23 @@ def __init__(self, filters, meta, template, sockets):
             os.makedirs(self.cachePath)
 
     def split(self, text, chunk_size, overlap , marker,  out):
-        tokens = self.pipe.tokenizer.tokenize(text)
-        for i in range(0, len(tokens), chunk_size-overlap):
-            chunk_tokens = tokens[i:i+chunk_size]
-            chunk = self.pipe.tokenizer.convert_tokens_to_string(chunk_tokens)
+        enc = tiktoken.get_encoding("cl100k_base")
+        tokenized_text = enc.encode(text)
+
+        for i in range(0, len(tokenized_text), chunk_size-overlap):
+            chunk_tokens = tokenized_text[i:min(i+chunk_size, len(tokenized_text))]
+            chunk = enc.decode(chunk_tokens)
             out.append([chunk, marker])
 
+
+
+
+        # tokens = self.pipe.tokenizer.tokenize(text)
+        # for i in range(0, len(tokens), chunk_size-overlap):
+        #     chunk_tokens = tokens[i:min(i+chunk_size, len(tokens))]
+        #     chunk = self.pipe.tokenizer.convert_tokens_to_string(chunk_tokens)
+        #     out.append([chunk, marker])
+
     def encode(self, sentences):       
         to_encode = []
         to_encode_index=[]
@@ -84,7 +95,7 @@ def getParamValue(key,default=None):
             if marker != "query": marker="passage"
             if data_type == "text":
                 sentences.append([data,marker])
-            elif datatype=="application/hyperblob":
+            elif data_type=="application/hyperblob":
                 blobDisk = self.openStorage(data)
                 files = blobDisk.list()
                 supportedExts = ["html","txt","htm","md"]