🚧 fix(wip): back to dedupe debug

Author: mxchinegod

magnet/electrode.py

@@ -2,41 +2,17 @@
 from magnet.ize import memory
 from magnet.ron import llm
 
+from magnet.utils.globals import _f
+
 class Electrode:
     def __init__(self, config: dict = None):
-        self.config = config if config else {
-            "MILVUS_URI": "192.168.2.69"
-            , "MILVUS_PORT": 19530
-            , "MILVUS_USER": "root"
-            , "MILVUS_PASSWORD": "Rrr/Yp6k#<M19rB3j1>Mi4Ta"
-            , "NATS_URL": "192.168.2.69"
-            , "NATS_USER": "my-user"
-            , "NATS_PASSWORD": "T0pS3cr3t"
-            , "NATS_CATEGORY": "non_nlp_chunks"
-            , "NATS_STREAM": "documents"
-            , "NATS_SESSION": "bge_large_en_v15"
-            , "DIMENSION": 1024
-            , "EMBEDDING_MODEL": "BAAI/bge-large-en-v1.5"
-            , "INDEX": "bge_non_nlp"
-            , "INDEX_PARAMS": {
-                'metric_type': 'COSINE',
-                'index_type':'HNSW',
-                'params': {
-                    "efConstruction": 40
-                    , "M": 48
-                },
-            }
-            , "JOB_TYPE": "index"
-            , "JOB_N": 10
-            , "GENERATION_MODEL": "mistralai/Mistral-7B-Instruct-v0.1"
-            , "CREATE": True
-        }
+        self.config = config if config else _f('fatal', 'no config applied')
     async def auto(self):
         match self.config['JOB_TYPE']:
             case 'index':
                 self.reso = field.Resonator(f"{self.config['NATS_USER']}:{self.config['NATS_PASSWORD']}@{self.config['NATS_URL']}")
                 self.embedder = memory.Embedder(self.config, create=self.config["CREATE"])
-                await self.reso.on(category=self.config['NATS_CATEGORY'], session=self.config['NATS_SESSION'], stream=self.config['NATS_STREAM'])
+                await self.reso.on(category=self.config['NATS_CATEGORY'], session=self.config['NATS_SESSION'], stream=self.config['NATS_STREAM'], job=True)
                 await self.reso.listen(cb=self.embedder.index, job_n=self.config['JOB_N'])
 
 

magnet/ic/field.py

@@ -298,6 +298,8 @@ async def worker(self, cb=print):
                 _f("warn", f'something wrong in your callback function!\n{e}')
         except Exception as e:
             _f('fatal','invalid JSON')
+    async def conduct(self, cb=print):
+        pass
 
     async def info(self, session: str = None):
         """

magnet/ize/memory.py

@@ -83,7 +83,7 @@ async def index(self, payload, msg, verbose=False, field=None, charge=False, ins
                             await self.field.pulse(payload)
                     await msg.ack_sync()
                 else:
-                    _f('info', 'embedding exists already') if verbose else None
+                    _f('info', f'embedding exists already\n{payload.text}')
                     await msg.ack_sync()
             except Exception as e:
                 await msg.term()
@@ -182,4 +182,4 @@ def is_dupe(self, q):
             , output_fields=['text', 'document']
             , limit=1
         )
-        return True if sum(match[0].distances) == 0.0 and len(match[0])>0 else False
+        return True if sum(match[0].distances) >= 0.99 and len(match[0])>0 else False