diff --git a/nesis/api/core/document_loaders/runners.py b/nesis/api/core/document_loaders/runners.py index f75e521..baf408c 100644 --- a/nesis/api/core/document_loaders/runners.py +++ b/nesis/api/core/document_loaders/runners.py @@ -183,31 +183,28 @@ def _is_modified( document: Document = get_document(document_id=document_id) if document is None or document.base_uri != endpoint: return False - elif document.base_uri == endpoint: - store_metadata = document.store_metadata - document_last_modified = document.last_modified - if ( - document_last_modified is None - and store_metadata - and store_metadata.get("last_modified") - ): - document_last_modified = strptime( - date_string=store_metadata["last_modified"] - ).replace(tzinfo=None) - if document_last_modified is not None and last_modified.replace( - microsecond=0 - ) > document_last_modified.replace(microsecond=0): - try: - self.delete(document=document, rag_metadata=document.rag_metadata) - except: - _LOG.warning( - f"Failed to delete document {document_id}'s record. Continuing anyway...", - exc_info=True, - ) - return True - return False - else: - return None + store_metadata = document.store_metadata + document_last_modified = document.last_modified + if ( + document_last_modified is None + and store_metadata is not None + and store_metadata.get("last_modified") + ): + document_last_modified = strptime( + date_string=store_metadata["last_modified"] + ).replace(tzinfo=None) + if document_last_modified is not None and last_modified.replace( + microsecond=0 + ) > document_last_modified.replace(microsecond=0): + try: + self.delete(document=document, rag_metadata=document.rag_metadata) + except: + _LOG.warning( + f"Failed to delete document {document_id}'s record. Continuing anyway...", + exc_info=True, + ) + return True + return False def save(self, **kwargs) -> Document: return save_document( @@ -231,10 +228,11 @@ def delete(self, document: Document, **kwargs) -> None: for document_data in rag_metadata.get("data") or [] ] ) - _LOG.info(f"Deleting document {document.filename}") - delete_document(document_id=document.id) except: _LOG.warning( f"Failed to delete document {document.filename}", exc_info=True, ) + + _LOG.info(f"Deleting document {document.filename}") + delete_document(document_id=document.id) diff --git a/nesis/rag/core/server/chat/chat_service.py b/nesis/rag/core/server/chat/chat_service.py index f339163..8044415 100644 --- a/nesis/rag/core/server/chat/chat_service.py +++ b/nesis/rag/core/server/chat/chat_service.py @@ -23,6 +23,7 @@ ) from nesis.rag.core.open_ai.extensions.context_filter import ContextFilter from nesis.rag.core.server.chunks.chunks_service import Chunk +from nesis.rag.core.settings.settings import Settings class Completion(BaseModel): @@ -78,8 +79,10 @@ def __init__( vector_store_component: VectorStoreComponent, embedding_component: EmbeddingComponent, node_store_component: NodeStoreComponent, + settings: Settings, ) -> None: self.llm_service = llm_component + self.settings = settings self.vector_store_component = vector_store_component self.storage_context = StorageContext.from_defaults( vector_store=vector_store_component.vector_store, @@ -104,7 +107,9 @@ def _chat_engine( ) -> BaseChatEngine: if use_context: vector_index_retriever = self.vector_store_component.get_retriever( - index=self.index, context_filter=context_filter + index=self.index, + context_filter=context_filter, + similarity_top_k=self.settings.vectorstore.similarity_top_k, ) return ContextChatEngine.from_defaults( system_prompt=system_prompt, diff --git a/nesis/rag/core/settings/settings.py b/nesis/rag/core/settings/settings.py index 3cbc761..d2014fb 100644 --- a/nesis/rag/core/settings/settings.py +++ b/nesis/rag/core/settings/settings.py @@ -110,6 +110,7 @@ class LLMSettings(BaseModel): class VectorstoreSettings(BaseModel): database: Literal["chroma", "qdrant", "pgvector"] + similarity_top_k: int class LocalSettings(BaseModel): diff --git a/nesis/rag/settings.yaml b/nesis/rag/settings.yaml index d2b4a7b..1d3c472 100644 --- a/nesis/rag/settings.yaml +++ b/nesis/rag/settings.yaml @@ -21,6 +21,7 @@ embedding: vectorstore: database: pgvector + similarity_top_k: ${NESIS_RAG_VECTORSTORE_SIMILARITY_TOP_K:5} pgvector: url: ${NESIS_RAG_PGVECTOR_URL:postgresql://postgres:password@localhost:65432/nesis} diff --git a/version.txt b/version.txt index 12023bf..80c2893 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.1.3-rc11 +0.1.3-rc23