Refactor ingest runner modified checker

ametnes · Jul 27, 2024 · fb73719 · fb73719
1 parent 8447207
commit fb73719
Show file tree

Hide file tree

Showing 5 changed files with 34 additions and 29 deletions.
diff --git a/nesis/api/core/document_loaders/runners.py b/nesis/api/core/document_loaders/runners.py
@@ -183,31 +183,28 @@ def _is_modified(
         document: Document = get_document(document_id=document_id)
         if document is None or document.base_uri != endpoint:
             return False
-        elif document.base_uri == endpoint:
-            store_metadata = document.store_metadata
-            document_last_modified = document.last_modified
-            if (
-                document_last_modified is None
-                and store_metadata
-                and store_metadata.get("last_modified")
-            ):
-                document_last_modified = strptime(
-                    date_string=store_metadata["last_modified"]
-                ).replace(tzinfo=None)
-            if document_last_modified is not None and last_modified.replace(
-                microsecond=0
-            ) > document_last_modified.replace(microsecond=0):
-                try:
-                    self.delete(document=document, rag_metadata=document.rag_metadata)
-                except:
-                    _LOG.warning(
-                        f"Failed to delete document {document_id}'s record. Continuing anyway...",
-                        exc_info=True,
-                    )
-                return True
-            return False
-        else:
-            return None
+        store_metadata = document.store_metadata
+        document_last_modified = document.last_modified
+        if (
+            document_last_modified is None
+            and store_metadata is not None
+            and store_metadata.get("last_modified")
+        ):
+            document_last_modified = strptime(
+                date_string=store_metadata["last_modified"]
+            ).replace(tzinfo=None)
+        if document_last_modified is not None and last_modified.replace(
+            microsecond=0
+        ) > document_last_modified.replace(microsecond=0):
+            try:
+                self.delete(document=document, rag_metadata=document.rag_metadata)
+            except:
+                _LOG.warning(
+                    f"Failed to delete document {document_id}'s record. Continuing anyway...",
+                    exc_info=True,
+                )
+            return True
+        return False
 
     def save(self, **kwargs) -> Document:
         return save_document(
@@ -231,10 +228,11 @@ def delete(self, document: Document, **kwargs) -> None:
                     for document_data in rag_metadata.get("data") or []
                 ]
             )
-            _LOG.info(f"Deleting document {document.filename}")
-            delete_document(document_id=document.id)
         except:
             _LOG.warning(
                 f"Failed to delete document {document.filename}",
                 exc_info=True,
             )
+
+        _LOG.info(f"Deleting document {document.filename}")
+        delete_document(document_id=document.id)
diff --git a/nesis/rag/core/server/chat/chat_service.py b/nesis/rag/core/server/chat/chat_service.py
@@ -23,6 +23,7 @@
 )
 from nesis.rag.core.open_ai.extensions.context_filter import ContextFilter
 from nesis.rag.core.server.chunks.chunks_service import Chunk
+from nesis.rag.core.settings.settings import Settings
 
 
 class Completion(BaseModel):
@@ -78,8 +79,10 @@ def __init__(
         vector_store_component: VectorStoreComponent,
         embedding_component: EmbeddingComponent,
         node_store_component: NodeStoreComponent,
+        settings: Settings,
     ) -> None:
         self.llm_service = llm_component
+        self.settings = settings
         self.vector_store_component = vector_store_component
         self.storage_context = StorageContext.from_defaults(
             vector_store=vector_store_component.vector_store,
@@ -104,7 +107,9 @@ def _chat_engine(
     ) -> BaseChatEngine:
         if use_context:
             vector_index_retriever = self.vector_store_component.get_retriever(
-                index=self.index, context_filter=context_filter
+                index=self.index,
+                context_filter=context_filter,
+                similarity_top_k=self.settings.vectorstore.similarity_top_k,
             )
             return ContextChatEngine.from_defaults(
                 system_prompt=system_prompt,

diff --git a/nesis/rag/core/settings/settings.py b/nesis/rag/core/settings/settings.py
@@ -110,6 +110,7 @@ class LLMSettings(BaseModel):
 
 class VectorstoreSettings(BaseModel):
     database: Literal["chroma", "qdrant", "pgvector"]
+    similarity_top_k: int
 
 
 class LocalSettings(BaseModel):

diff --git a/nesis/rag/settings.yaml b/nesis/rag/settings.yaml
@@ -21,6 +21,7 @@ embedding:
 
 vectorstore:
   database: pgvector
+  similarity_top_k: ${NESIS_RAG_VECTORSTORE_SIMILARITY_TOP_K:5}
 
 pgvector:
   url: ${NESIS_RAG_PGVECTOR_URL:postgresql://postgres:password@localhost:65432/nesis}

diff --git a/version.txt b/version.txt
@@ -1 +1 @@
-0.1.3-rc11
+0.1.3-rc23