From fa765f7825d9725abb52f239bcb806340c124522 Mon Sep 17 00:00:00 2001 From: Puneet Saraswat Date: Sun, 10 Dec 2023 09:26:39 +0530 Subject: [PATCH] reorganize configs, cleanups and add a workflow config --- querent/collectors/aws/aws_collector.py | 2 +- querent/collectors/azure/azure_collector.py | 2 +- querent/collectors/collector_factory.py | 2 +- querent/collectors/collector_resolver.py | 2 +- .../drive/google_drive_collector.py | 2 +- .../collectors/dropbox/dropbox_collector.py | 4 +- querent/collectors/email/email_collector.py | 2 +- querent/collectors/email/imap.py | 2 +- querent/collectors/fs/fs_collector.py | 2 +- querent/collectors/gcs/gcs_collector.py | 4 +- querent/collectors/github/github_collector.py | 2 +- querent/collectors/jira/jira_collector.py | 2 +- querent/collectors/slack/slack_collector.py | 2 +- .../webscaper/web_scraper_collector.py | 2 +- querent/config/collector/__init__.py | 0 .../{ => collector}/collector_config.py | 13 ++++-- querent/config/engine/__init__.py | 0 querent/config/{ => engine}/engine_config.py | 12 +++++- querent/config/ingestor/__init__.py | 0 .../config/{ => ingestor}/ingestor_config.py | 0 querent/config/schema_config.py | 13 ------ querent/config/storage_config.py | 41 ------------------- querent/config/workflow/__init__.py | 0 querent/config/workflow/workflow_config.py | 21 ++++++++++ querent/core/base_engine.py | 2 +- querent/ingestors/audio/audio_ingestors.py | 2 +- querent/ingestors/code/code_ingestor.py | 2 +- querent/ingestors/csv/csv_ingestor.py | 2 +- querent/ingestors/doc/doc_ingestor.py | 2 +- querent/ingestors/github/github_ingestor.py | 2 +- querent/ingestors/html/html_ingestor.py | 2 +- querent/ingestors/images/image_ingestor.py | 2 +- querent/ingestors/ingestor_manager.py | 2 +- querent/ingestors/json/json_ingestor.py | 2 +- querent/ingestors/pdfs/pdf_ingestor_v1.py | 2 +- querent/ingestors/ppt/ppt_ingestor.py | 2 +- querent/ingestors/texts/text_ingestor.py | 2 +- querent/ingestors/video/video_ingestor.py | 2 +- querent/ingestors/xlsx/xlsx_ingestor.py | 2 +- querent/ingestors/xml/xml_ingestor.py | 2 +- .../collector_tests/test_github_collector.py | 2 +- tests/collector_tests/test_local_collector.py | 2 +- tests/collector_tests/test_slack_collector.py | 2 +- tests/sad_tests/test_audio_error.py | 2 +- tests/sad_tests/test_csv_error.py | 2 +- tests/sad_tests/test_doc_ingestor.py | 2 +- tests/sad_tests/test_html_ingestor_error.py | 2 +- tests/sad_tests/test_image_ingestor_error.py | 2 +- tests/sad_tests/test_ppt_ingestor_error.py | 2 +- tests/sad_tests/test_txt_ingestor_error.py | 2 +- tests/test-fanning-in-collectors.py | 2 +- tests/test_async_collectors_ingestion.py | 2 +- tests/test_audio_ingestor.py | 2 +- tests/test_aws_collector.py | 4 +- tests/test_code_ingestor.py | 2 +- tests/test_csv_ingestor.py | 2 +- tests/test_doc_ingestor.py | 2 +- tests/test_drive_collector.py | 2 +- tests/test_dropbox_collector.py | 2 +- tests/test_email_collector.py | 2 +- tests/test_gcs_collector.py | 2 +- tests/test_generic_ingestor.py | 2 +- tests/test_github_ingestor.py | 2 +- tests/test_html_ingestor.py | 2 +- tests/test_image_ingestor.py | 2 +- tests/test_jira_collector.py | 2 +- tests/test_json_ingestor.py | 2 +- tests/test_pdf_ingestor.py | 2 +- tests/test_ppt_ingestor.py | 2 +- tests/test_text_ingestor.py | 2 +- tests/test_webscrapper.py | 2 +- tests/test_xlsx_ingestor.py | 2 +- tests/test_xml_ingestor.py | 2 +- 73 files changed, 107 insertions(+), 125 deletions(-) create mode 100644 querent/config/collector/__init__.py rename querent/config/{ => collector}/collector_config.py (89%) create mode 100644 querent/config/engine/__init__.py rename querent/config/{ => engine}/engine_config.py (56%) create mode 100644 querent/config/ingestor/__init__.py rename querent/config/{ => ingestor}/ingestor_config.py (100%) delete mode 100644 querent/config/schema_config.py delete mode 100644 querent/config/storage_config.py create mode 100644 querent/config/workflow/__init__.py create mode 100644 querent/config/workflow/workflow_config.py diff --git a/querent/collectors/aws/aws_collector.py b/querent/collectors/aws/aws_collector.py index 8c7a1d95..5ac2c887 100644 --- a/querent/collectors/aws/aws_collector.py +++ b/querent/collectors/aws/aws_collector.py @@ -6,7 +6,7 @@ from querent.common.types.collected_bytes import ( CollectedBytes, ) # Import for handling authentication errors -from querent.config.collector_config import CollectorBackend, S3CollectConfig +from querent.config.collector.collector_config import CollectorBackend, S3CollectConfig from querent.collectors.collector_base import Collector from querent.collectors.collector_factory import CollectorFactory from querent.common.uri import Uri diff --git a/querent/collectors/azure/azure_collector.py b/querent/collectors/azure/azure_collector.py index d6732928..2154a75e 100644 --- a/querent/collectors/azure/azure_collector.py +++ b/querent/collectors/azure/azure_collector.py @@ -3,7 +3,7 @@ from azure.storage.blob import BlobServiceClient from querent.common.types.collected_bytes import CollectedBytes -from querent.config.collector_config import CollectorBackend, AzureCollectConfig +from querent.config.collector.collector_config import CollectorBackend, AzureCollectConfig from querent.collectors.collector_base import Collector from querent.collectors.collector_factory import CollectorFactory from querent.common.uri import Uri diff --git a/querent/collectors/collector_factory.py b/querent/collectors/collector_factory.py index a4fd77a3..a2424fdf 100644 --- a/querent/collectors/collector_factory.py +++ b/querent/collectors/collector_factory.py @@ -6,7 +6,7 @@ CollectorResolverError, CollectorErrorKind, ) -from querent.config.collector_config import CollectorConfig, CollectorBackend +from querent.config.collector.collector_config import CollectorConfig, CollectorBackend class CollectorFactory(ABC): diff --git a/querent/collectors/collector_resolver.py b/querent/collectors/collector_resolver.py index 6256d19b..6d2a3496 100644 --- a/querent/collectors/collector_resolver.py +++ b/querent/collectors/collector_resolver.py @@ -10,7 +10,7 @@ from querent.collectors.dropbox.dropbox_collector import DropBoxCollectorFactory from querent.collectors.github.github_collector import GithubCollectorFactory from querent.collectors.drive.google_drive_collector import DriveCollectorFactory -from querent.config.collector_config import CollectorConfig, CollectorBackend +from querent.config.collector.collector_config import CollectorConfig, CollectorBackend from querent.collectors.collector_base import Collector from querent.collectors.collector_errors import ( CollectorResolverError, diff --git a/querent/collectors/drive/google_drive_collector.py b/querent/collectors/drive/google_drive_collector.py index 70685645..b570f4a8 100644 --- a/querent/collectors/drive/google_drive_collector.py +++ b/querent/collectors/drive/google_drive_collector.py @@ -12,7 +12,7 @@ from querent.collectors.collector_factory import CollectorFactory from querent.common.types.collected_bytes import CollectedBytes from querent.common.uri import Uri -from querent.config.collector_config import CollectorBackend, DriveCollectorConfig +from querent.config.collector.collector_config import CollectorBackend, DriveCollectorConfig from querent.common import common_errors import requests diff --git a/querent/collectors/dropbox/dropbox_collector.py b/querent/collectors/dropbox/dropbox_collector.py index e1d258bc..7fcbd38f 100644 --- a/querent/collectors/dropbox/dropbox_collector.py +++ b/querent/collectors/dropbox/dropbox_collector.py @@ -3,9 +3,9 @@ import dropbox from dropbox.oauth import DropboxOAuth2FlowNoRedirect from querent.common.types.collected_bytes import CollectedBytes -from querent.config.collector_config import DropboxConfig +from querent.config.collector.collector_config import DropboxConfig from querent.collectors.collector_base import Collector -from querent.config.collector_config import CollectorBackend +from querent.config.collector.collector_config import CollectorBackend from querent.collectors.collector_factory import CollectorFactory from querent.common import common_errors from querent.common.uri import Uri diff --git a/querent/collectors/email/email_collector.py b/querent/collectors/email/email_collector.py index 89850467..2fc6c7cc 100644 --- a/querent/collectors/email/email_collector.py +++ b/querent/collectors/email/email_collector.py @@ -8,7 +8,7 @@ from querent.common import common_errors from querent.common.types.collected_bytes import CollectedBytes from querent.common.uri import Uri -from querent.config.collector_config import CollectorBackend, EmailCollectorConfig +from querent.config.collector.collector_config import CollectorBackend, EmailCollectorConfig from querent.logging.logger import setup_logger diff --git a/querent/collectors/email/imap.py b/querent/collectors/email/imap.py index 81c8c498..81137c84 100644 --- a/querent/collectors/email/imap.py +++ b/querent/collectors/email/imap.py @@ -1,6 +1,6 @@ import imaplib -from querent.config.collector_config import EmailCollectorConfig +from querent.config.collector.collector_config import EmailCollectorConfig class ImapEmail: diff --git a/querent/collectors/fs/fs_collector.py b/querent/collectors/fs/fs_collector.py index 6ac810d5..f85b8936 100644 --- a/querent/collectors/fs/fs_collector.py +++ b/querent/collectors/fs/fs_collector.py @@ -7,7 +7,7 @@ from querent.collectors.collector_factory import CollectorFactory from querent.common.types.collected_bytes import CollectedBytes from querent.common.uri import Uri -from querent.config.collector_config import CollectorBackend, FSCollectorConfig +from querent.config.collector.collector_config import CollectorBackend, FSCollectorConfig import aiofiles from querent.common import common_errors diff --git a/querent/collectors/gcs/gcs_collector.py b/querent/collectors/gcs/gcs_collector.py index 32f45473..0eafa226 100644 --- a/querent/collectors/gcs/gcs_collector.py +++ b/querent/collectors/gcs/gcs_collector.py @@ -3,8 +3,8 @@ import aiofiles from querent.common.types.collected_bytes import CollectedBytes -from querent.config.collector_config import GcsCollectConfig -from querent.config.collector_config import CollectorBackend +from querent.config.collector.collector_config import GcsCollectConfig +from querent.config.collector.collector_config import CollectorBackend from querent.collectors.collector_base import Collector from querent.collectors.collector_factory import CollectorFactory from querent.common.uri import Uri diff --git a/querent/collectors/github/github_collector.py b/querent/collectors/github/github_collector.py index 44c38232..af368d75 100644 --- a/querent/collectors/github/github_collector.py +++ b/querent/collectors/github/github_collector.py @@ -3,7 +3,7 @@ from querent.common.types.collected_bytes import CollectedBytes -from querent.config.collector_config import GithubConfig, CollectorBackend +from querent.config.collector.collector_config import GithubConfig, CollectorBackend from querent.collectors.collector_base import Collector from querent.collectors.collector_factory import CollectorFactory from querent.common.uri import Uri diff --git a/querent/collectors/jira/jira_collector.py b/querent/collectors/jira/jira_collector.py index 1393e64b..9562af1d 100644 --- a/querent/collectors/jira/jira_collector.py +++ b/querent/collectors/jira/jira_collector.py @@ -6,7 +6,7 @@ from querent.common import common_errors from querent.common.types.collected_bytes import CollectedBytes from querent.common.uri import Uri -from querent.config.collector_config import CollectorBackend, JiraCollectorConfig +from querent.config.collector.collector_config import CollectorBackend, JiraCollectorConfig from jira import JIRA from querent.logging.logger import setup_logger diff --git a/querent/collectors/slack/slack_collector.py b/querent/collectors/slack/slack_collector.py index fb338db6..b1c2eed7 100644 --- a/querent/collectors/slack/slack_collector.py +++ b/querent/collectors/slack/slack_collector.py @@ -4,7 +4,7 @@ import os from querent.collectors.collector_factory import CollectorFactory -from querent.config.collector_config import CollectorBackend, SlackCollectorConfig +from querent.config.collector.collector_config import CollectorBackend, SlackCollectorConfig from querent.collectors.collector_base import Collector from querent.common.types.collected_bytes import CollectedBytes from querent.common.uri import Uri diff --git a/querent/collectors/webscaper/web_scraper_collector.py b/querent/collectors/webscaper/web_scraper_collector.py index d4b3e027..e53ab375 100644 --- a/querent/collectors/webscaper/web_scraper_collector.py +++ b/querent/collectors/webscaper/web_scraper_collector.py @@ -3,7 +3,7 @@ from querent.collectors.collector_base import Collector from querent.collectors.collector_factory import CollectorFactory from querent.common.types.collected_bytes import CollectedBytes -from querent.config.collector_config import CollectorBackend, WebScraperConfig +from querent.config.collector.collector_config import CollectorBackend, WebScraperConfig from querent.common.uri import Uri from querent.tools.web_page_extractor import WebpageExtractor from urllib.parse import urlparse, urljoin diff --git a/querent/config/collector/__init__.py b/querent/config/collector/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/querent/config/collector_config.py b/querent/config/collector/collector_config.py similarity index 89% rename from querent/config/collector_config.py rename to querent/config/collector/collector_config.py index c8daba18..a79155e9 100644 --- a/querent/config/collector_config.py +++ b/querent/config/collector/collector_config.py @@ -1,6 +1,6 @@ from enum import Enum -from typing import List, Optional, Union -from pydantic import BaseModel, Field +from typing import Any, List, Optional, Union +from pydantic import BaseModel, Field, validator from querent.channel.channel_interface import ChannelCommandInterface @@ -21,7 +21,14 @@ class CollectorBackend(str, Enum): class CollectorConfig(BaseModel): backend: CollectorBackend - channel: ChannelCommandInterface + # Use Field with allow_mutation=False to specify the type + channel: Any + + # Custom validator for ChannelCommandInterface + @validator("channel", pre=True, allow_reuse=True) + def validate_channel(cls, value): + # Perform any additional validation logic here + return value class FSCollectorConfig(CollectorConfig): diff --git a/querent/config/engine/__init__.py b/querent/config/engine/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/querent/config/engine_config.py b/querent/config/engine/engine_config.py similarity index 56% rename from querent/config/engine_config.py rename to querent/config/engine/engine_config.py index 304efe0a..a2b79e90 100644 --- a/querent/config/engine_config.py +++ b/querent/config/engine/engine_config.py @@ -1,4 +1,5 @@ -from pydantic import BaseModel +from typing import Any +from pydantic import BaseModel, Field, validator from querent.channel.channel_interface import ChannelCommandInterface @@ -15,7 +16,14 @@ class EngineConfig(BaseModel): retry_interval: float = 2.0 message_throttle_limit: int = 1000 message_throttle_delay: float = 0.001 - channel: ChannelCommandInterface + # Use Field with allow_mutation=False to specify the type + channel: Any logger: str = f"{__name__}.engine_config" state_queue: str = f"{__name__}.state_queue" workers: str = f"{__name__}.workers" + + # Custom validator for ChannelCommandInterface + @validator("channel", pre=True, allow_reuse=True) + def validate_channel(cls, value): + # Perform any additional validation logic here + return value diff --git a/querent/config/ingestor/__init__.py b/querent/config/ingestor/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/querent/config/ingestor_config.py b/querent/config/ingestor/ingestor_config.py similarity index 100% rename from querent/config/ingestor_config.py rename to querent/config/ingestor/ingestor_config.py diff --git a/querent/config/schema_config.py b/querent/config/schema_config.py deleted file mode 100644 index 5f96dec6..00000000 --- a/querent/config/schema_config.py +++ /dev/null @@ -1,13 +0,0 @@ -from pydantic import BaseModel - - -class GraphSchemaConfig(BaseModel): - """ - Schema config for a knowledge graph object. - """ - - name: str - schema_content: str - shacl_content: str - schema_format: str - shacl_format: str \ No newline at end of file diff --git a/querent/config/storage_config.py b/querent/config/storage_config.py deleted file mode 100644 index 7f60f31c..00000000 --- a/querent/config/storage_config.py +++ /dev/null @@ -1,41 +0,0 @@ -from enum import Enum -from typing import Optional -from pydantic import BaseModel - -class StorageBackend(str, Enum): - LocalFile = "localfile" - Redis = "redis" - -class StorageBackendFlavor(str, Enum): - DigitalOcean = "do" - Garage = "garage" - Gcs = "gcp" - MinIO = "minio" - -class StorageConfig(BaseModel): - backend: StorageBackend - flavor: Optional[StorageBackendFlavor] = None - - class Config: - use_enum_values = True - -class LocalFileStorageConfig(BaseModel): - root_path: str - -class RedisStorageConfig(BaseModel): - host: str - port: int - password: Optional[str] = None - -class StorageConfigWrapper(BaseModel): - backend: StorageBackend - config: Optional[BaseModel] = None - - @classmethod - def from_storage_config(cls, storage_config: StorageConfig): - if storage_config.backend == StorageBackend.LocalFile: - return cls(backend=StorageBackend.LocalFile, config=LocalFileStorageConfig()) - elif storage_config.backend == StorageBackend.Redis: - raise NotImplementedError("Redis storage configuration is not implemented yet") - else: - raise ValueError(f"Unsupported storage backend: {storage_config.backend}") diff --git a/querent/config/workflow/__init__.py b/querent/config/workflow/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/querent/config/workflow/workflow_config.py b/querent/config/workflow/workflow_config.py new file mode 100644 index 00000000..29fcbad7 --- /dev/null +++ b/querent/config/workflow/workflow_config.py @@ -0,0 +1,21 @@ +from typing import Any +from pydantic import BaseModel, validator + + +class WorkflowConfig(BaseModel): + """Workflow configuration.""" + + name: str + id: str + channel: Any + event_handler: Any + + @validator("channel", pre=True, allow_reuse=True) + def validate_channel(cls, value): + # Perform any additional validation logic here + return value + + @validator("event_handler", pre=True, allow_reuse=True) + def validate_event_handler(cls, value): + # Perform any additional validation logic here + return value diff --git a/querent/core/base_engine.py b/querent/core/base_engine.py index 156e12fc..df62a65a 100644 --- a/querent/core/base_engine.py +++ b/querent/core/base_engine.py @@ -7,7 +7,7 @@ from querent.common.types.ingested_code import IngestedCode from querent.common.types.querent_event import EventState, EventType from querent.common.types.querent_queue import QuerentQueue -from querent.config.engine_config import EngineConfig +from querent.config.engine.engine_config import EngineConfig from querent.logging.logger import setup_logger """ diff --git a/querent/ingestors/audio/audio_ingestors.py b/querent/ingestors/audio/audio_ingestors.py index 078b8cbc..4b332dab 100644 --- a/querent/ingestors/audio/audio_ingestors.py +++ b/querent/ingestors/audio/audio_ingestors.py @@ -6,7 +6,7 @@ from querent.ingestors.ingestor_factory import IngestorFactory from querent.processors.async_processor import AsyncProcessor from querent.ingestors.base_ingestor import BaseIngestor -from querent.config.ingestor_config import IngestorBackend +from querent.config.ingestor.ingestor_config import IngestorBackend from querent.common.types.collected_bytes import CollectedBytes from querent.common.common_errors import ( UnknownValueError, diff --git a/querent/ingestors/code/code_ingestor.py b/querent/ingestors/code/code_ingestor.py index 2f0903d2..38940015 100644 --- a/querent/ingestors/code/code_ingestor.py +++ b/querent/ingestors/code/code_ingestor.py @@ -3,7 +3,7 @@ from querent.ingestors.base_ingestor import BaseIngestor from querent.ingestors.ingestor_factory import IngestorFactory from querent.processors.async_processor import AsyncProcessor -from querent.config.ingestor_config import IngestorBackend +from querent.config.ingestor.ingestor_config import IngestorBackend from querent.common import common_errors from querent.common.types.ingested_tokens import IngestedTokens diff --git a/querent/ingestors/csv/csv_ingestor.py b/querent/ingestors/csv/csv_ingestor.py index 245ef973..44befbfa 100644 --- a/querent/ingestors/csv/csv_ingestor.py +++ b/querent/ingestors/csv/csv_ingestor.py @@ -5,7 +5,7 @@ from querent.processors.async_processor import AsyncProcessor from querent.ingestors.ingestor_factory import IngestorFactory from querent.ingestors.base_ingestor import BaseIngestor -from querent.config.ingestor_config import IngestorBackend +from querent.config.ingestor.ingestor_config import IngestorBackend from querent.common.types.collected_bytes import CollectedBytes from querent.common import common_errors from querent.common.types.ingested_tokens import IngestedTokens diff --git a/querent/ingestors/doc/doc_ingestor.py b/querent/ingestors/doc/doc_ingestor.py index 81d4fa24..7e7ee96f 100644 --- a/querent/ingestors/doc/doc_ingestor.py +++ b/querent/ingestors/doc/doc_ingestor.py @@ -7,7 +7,7 @@ from querent.processors.async_processor import AsyncProcessor from querent.ingestors.ingestor_factory import IngestorFactory from querent.ingestors.base_ingestor import BaseIngestor -from querent.config.ingestor_config import IngestorBackend +from querent.config.ingestor.ingestor_config import IngestorBackend from querent.common.types.collected_bytes import CollectedBytes from querent.common import common_errors from querent.common.types.ingested_tokens import IngestedTokens diff --git a/querent/ingestors/github/github_ingestor.py b/querent/ingestors/github/github_ingestor.py index eca8c3e1..8c1df80f 100644 --- a/querent/ingestors/github/github_ingestor.py +++ b/querent/ingestors/github/github_ingestor.py @@ -3,7 +3,7 @@ from querent.ingestors.base_ingestor import BaseIngestor from querent.ingestors.ingestor_factory import IngestorFactory from querent.processors.async_processor import AsyncProcessor -from querent.config.ingestor_config import IngestorBackend +from querent.config.ingestor.ingestor_config import IngestorBackend from querent.common.types.ingested_code import IngestedCode from querent.common.types.collected_bytes import CollectedBytes diff --git a/querent/ingestors/html/html_ingestor.py b/querent/ingestors/html/html_ingestor.py index a43fd74c..4cd7820f 100644 --- a/querent/ingestors/html/html_ingestor.py +++ b/querent/ingestors/html/html_ingestor.py @@ -4,7 +4,7 @@ from querent.processors.async_processor import AsyncProcessor from querent.ingestors.ingestor_factory import IngestorFactory from querent.ingestors.base_ingestor import BaseIngestor -from querent.config.ingestor_config import IngestorBackend +from querent.config.ingestor.ingestor_config import IngestorBackend from querent.common.types.collected_bytes import CollectedBytes from querent.common import common_errors from querent.common.types.ingested_tokens import IngestedTokens diff --git a/querent/ingestors/images/image_ingestor.py b/querent/ingestors/images/image_ingestor.py index e2fef8c1..f29ada75 100644 --- a/querent/ingestors/images/image_ingestor.py +++ b/querent/ingestors/images/image_ingestor.py @@ -3,7 +3,7 @@ from querent.ingestors.base_ingestor import BaseIngestor from querent.ingestors.ingestor_factory import IngestorFactory from querent.processors.async_processor import AsyncProcessor -from querent.config.ingestor_config import IngestorBackend +from querent.config.ingestor.ingestor_config import IngestorBackend from querent.processors.async_processor import AsyncProcessor from querent.common.common_errors import ( FileNotFoundError, diff --git a/querent/ingestors/ingestor_manager.py b/querent/ingestors/ingestor_manager.py index d0c3785b..e7bca426 100644 --- a/querent/ingestors/ingestor_manager.py +++ b/querent/ingestors/ingestor_manager.py @@ -6,7 +6,7 @@ from querent.collectors.collector_base import Collector from querent.common.types.collected_bytes import CollectedBytes from querent.common.types.ingested_tokens import IngestedTokens -from querent.config.ingestor_config import IngestorBackend +from querent.config.ingestor.ingestor_config import IngestorBackend from querent.ingestors.base_ingestor import BaseIngestor from querent.ingestors.ingestor_factory import IngestorFactory, UnsupportedIngestor from querent.ingestors.pdfs.pdf_ingestor_v1 import PdfIngestorFactory diff --git a/querent/ingestors/json/json_ingestor.py b/querent/ingestors/json/json_ingestor.py index 68d4b3e2..0d921425 100644 --- a/querent/ingestors/json/json_ingestor.py +++ b/querent/ingestors/json/json_ingestor.py @@ -1,7 +1,7 @@ from typing import AsyncGenerator, List import json from querent.common.types.collected_bytes import CollectedBytes -from querent.config.ingestor_config import IngestorBackend +from querent.config.ingestor.ingestor_config import IngestorBackend from querent.ingestors.base_ingestor import BaseIngestor from querent.ingestors.ingestor_factory import IngestorFactory from querent.processors.async_processor import AsyncProcessor diff --git a/querent/ingestors/pdfs/pdf_ingestor_v1.py b/querent/ingestors/pdfs/pdf_ingestor_v1.py index 94cc3ba8..a4a361fa 100644 --- a/querent/ingestors/pdfs/pdf_ingestor_v1.py +++ b/querent/ingestors/pdfs/pdf_ingestor_v1.py @@ -3,7 +3,7 @@ from querent.common.types.collected_bytes import CollectedBytes from querent.common.types.ingested_tokens import IngestedTokens from querent.common.types.ingested_images import IngestedImages -from querent.config.ingestor_config import IngestorBackend +from querent.config.ingestor.ingestor_config import IngestorBackend from querent.ingestors.base_ingestor import BaseIngestor from querent.ingestors.ingestor_factory import IngestorFactory from querent.processors.async_processor import AsyncProcessor diff --git a/querent/ingestors/ppt/ppt_ingestor.py b/querent/ingestors/ppt/ppt_ingestor.py index 3d1cbb31..410fc56e 100644 --- a/querent/ingestors/ppt/ppt_ingestor.py +++ b/querent/ingestors/ppt/ppt_ingestor.py @@ -8,7 +8,7 @@ from querent.ingestors.ingestor_factory import IngestorFactory from querent.processors.async_processor import AsyncProcessor from querent.ingestors.base_ingestor import BaseIngestor -from querent.config.ingestor_config import IngestorBackend +from querent.config.ingestor.ingestor_config import IngestorBackend from querent.common.types.collected_bytes import CollectedBytes from querent.common import common_errors from querent.common.types.ingested_tokens import IngestedTokens diff --git a/querent/ingestors/texts/text_ingestor.py b/querent/ingestors/texts/text_ingestor.py index cbb9984f..a9788d5c 100644 --- a/querent/ingestors/texts/text_ingestor.py +++ b/querent/ingestors/texts/text_ingestor.py @@ -3,7 +3,7 @@ from querent.ingestors.base_ingestor import BaseIngestor from querent.ingestors.ingestor_factory import IngestorFactory from querent.processors.async_processor import AsyncProcessor -from querent.config.ingestor_config import IngestorBackend +from querent.config.ingestor.ingestor_config import IngestorBackend from querent.common import common_errors from querent.common.types.ingested_tokens import IngestedTokens diff --git a/querent/ingestors/video/video_ingestor.py b/querent/ingestors/video/video_ingestor.py index 3362de5c..d70a8128 100644 --- a/querent/ingestors/video/video_ingestor.py +++ b/querent/ingestors/video/video_ingestor.py @@ -4,7 +4,7 @@ from querent.ingestors.ingestor_factory import IngestorFactory from querent.processors.async_processor import AsyncProcessor from querent.ingestors.base_ingestor import BaseIngestor -from querent.config.ingestor_config import IngestorBackend +from querent.config.ingestor.ingestor_config import IngestorBackend from querent.common.types.collected_bytes import CollectedBytes from querent.common.types.ingested_tokens import IngestedTokens import moviepy.editor as mp diff --git a/querent/ingestors/xlsx/xlsx_ingestor.py b/querent/ingestors/xlsx/xlsx_ingestor.py index a3b22943..27a3e8b1 100644 --- a/querent/ingestors/xlsx/xlsx_ingestor.py +++ b/querent/ingestors/xlsx/xlsx_ingestor.py @@ -5,7 +5,7 @@ from querent.ingestors.ingestor_factory import IngestorFactory from querent.ingestors.base_ingestor import BaseIngestor from querent.processors.async_processor import AsyncProcessor -from querent.config.ingestor_config import IngestorBackend +from querent.config.ingestor.ingestor_config import IngestorBackend from querent.common.types.collected_bytes import CollectedBytes from querent.common.types.ingested_tokens import ( IngestedTokens, diff --git a/querent/ingestors/xml/xml_ingestor.py b/querent/ingestors/xml/xml_ingestor.py index 5270b803..6eea3cad 100644 --- a/querent/ingestors/xml/xml_ingestor.py +++ b/querent/ingestors/xml/xml_ingestor.py @@ -4,7 +4,7 @@ from querent.processors.async_processor import AsyncProcessor from querent.ingestors.ingestor_factory import IngestorFactory -from querent.config.ingestor_config import IngestorBackend +from querent.config.ingestor.ingestor_config import IngestorBackend from querent.ingestors.base_ingestor import BaseIngestor from querent.common.types.collected_bytes import CollectedBytes from querent.common.types.ingested_tokens import ( diff --git a/tests/collector_tests/test_github_collector.py b/tests/collector_tests/test_github_collector.py index 9cb4be07..887a0197 100644 --- a/tests/collector_tests/test_github_collector.py +++ b/tests/collector_tests/test_github_collector.py @@ -2,7 +2,7 @@ import pytest import os from querent.collectors.collector_resolver import CollectorResolver -from querent.config.collector_config import GithubConfig +from querent.config.collector.collector_config import GithubConfig from querent.common.uri import Uri import uuid from dotenv import load_dotenv diff --git a/tests/collector_tests/test_local_collector.py b/tests/collector_tests/test_local_collector.py index ccd4bd44..71334546 100644 --- a/tests/collector_tests/test_local_collector.py +++ b/tests/collector_tests/test_local_collector.py @@ -7,7 +7,7 @@ import uuid from querent.common.uri import Uri -from querent.config.collector_config import CollectorBackend, FSCollectorConfig +from querent.config.collector.collector_config import CollectorBackend, FSCollectorConfig @pytest.fixture diff --git a/tests/collector_tests/test_slack_collector.py b/tests/collector_tests/test_slack_collector.py index a3b15ab9..9ab5d93b 100644 --- a/tests/collector_tests/test_slack_collector.py +++ b/tests/collector_tests/test_slack_collector.py @@ -2,7 +2,7 @@ import pytest import os from querent.collectors.collector_resolver import CollectorResolver -from querent.config.collector_config import SlackCollectorConfig +from querent.config.collector.collector_config import SlackCollectorConfig from querent.common.uri import Uri import uuid diff --git a/tests/sad_tests/test_audio_error.py b/tests/sad_tests/test_audio_error.py index de98415e..bc9b911c 100644 --- a/tests/sad_tests/test_audio_error.py +++ b/tests/sad_tests/test_audio_error.py @@ -1,7 +1,7 @@ import asyncio from pathlib import Path from querent.collectors.fs.fs_collector import FSCollectorFactory -from querent.config.collector_config import FSCollectorConfig +from querent.config.collector.collector_config import FSCollectorConfig from querent.common.uri import Uri from querent.ingestors.ingestor_manager import IngestorFactoryManager from querent.common import common_errors diff --git a/tests/sad_tests/test_csv_error.py b/tests/sad_tests/test_csv_error.py index ab79f729..ed70dfa0 100644 --- a/tests/sad_tests/test_csv_error.py +++ b/tests/sad_tests/test_csv_error.py @@ -1,7 +1,7 @@ import asyncio from pathlib import Path from querent.collectors.fs.fs_collector import FSCollectorFactory -from querent.config.collector_config import FSCollectorConfig +from querent.config.collector.collector_config import FSCollectorConfig from querent.common.uri import Uri from querent.ingestors.ingestor_manager import IngestorFactoryManager from querent.common import common_errors diff --git a/tests/sad_tests/test_doc_ingestor.py b/tests/sad_tests/test_doc_ingestor.py index a7a0f342..3b7e111f 100644 --- a/tests/sad_tests/test_doc_ingestor.py +++ b/tests/sad_tests/test_doc_ingestor.py @@ -1,7 +1,7 @@ import asyncio from pathlib import Path from querent.collectors.fs.fs_collector import FSCollectorFactory -from querent.config.collector_config import FSCollectorConfig +from querent.config.collector.collector_config import FSCollectorConfig from querent.common.uri import Uri from querent.ingestors.ingestor_manager import IngestorFactoryManager from querent.common.common_errors import UnknownError diff --git a/tests/sad_tests/test_html_ingestor_error.py b/tests/sad_tests/test_html_ingestor_error.py index f7e8c725..c353bef5 100644 --- a/tests/sad_tests/test_html_ingestor_error.py +++ b/tests/sad_tests/test_html_ingestor_error.py @@ -1,7 +1,7 @@ import asyncio from pathlib import Path from querent.collectors.fs.fs_collector import FSCollectorFactory -from querent.config.collector_config import FSCollectorConfig +from querent.config.collector.collector_config import FSCollectorConfig from querent.common.uri import Uri from querent.ingestors.ingestor_manager import IngestorFactoryManager from querent.common import common_errors diff --git a/tests/sad_tests/test_image_ingestor_error.py b/tests/sad_tests/test_image_ingestor_error.py index 176fa971..41c307ff 100644 --- a/tests/sad_tests/test_image_ingestor_error.py +++ b/tests/sad_tests/test_image_ingestor_error.py @@ -1,7 +1,7 @@ import asyncio from pathlib import Path from querent.collectors.fs.fs_collector import FSCollectorFactory -from querent.config.collector_config import FSCollectorConfig +from querent.config.collector.collector_config import FSCollectorConfig from querent.common.uri import Uri from querent.ingestors.ingestor_manager import IngestorFactoryManager import pytest diff --git a/tests/sad_tests/test_ppt_ingestor_error.py b/tests/sad_tests/test_ppt_ingestor_error.py index dfb1d448..234b23b3 100644 --- a/tests/sad_tests/test_ppt_ingestor_error.py +++ b/tests/sad_tests/test_ppt_ingestor_error.py @@ -1,7 +1,7 @@ import asyncio from pathlib import Path from querent.collectors.fs.fs_collector import FSCollectorFactory -from querent.config.collector_config import FSCollectorConfig +from querent.config.collector.collector_config import FSCollectorConfig from querent.common.uri import Uri from querent.ingestors.ingestor_manager import IngestorFactoryManager from querent.common import common_errors diff --git a/tests/sad_tests/test_txt_ingestor_error.py b/tests/sad_tests/test_txt_ingestor_error.py index 5ae257e3..e77cf8c3 100644 --- a/tests/sad_tests/test_txt_ingestor_error.py +++ b/tests/sad_tests/test_txt_ingestor_error.py @@ -1,7 +1,7 @@ import asyncio from pathlib import Path from querent.collectors.fs.fs_collector import FSCollectorFactory -from querent.config.collector_config import FSCollectorConfig +from querent.config.collector.collector_config import FSCollectorConfig from querent.common.uri import Uri from querent.ingestors.ingestor_manager import IngestorFactoryManager from querent.common import common_errors diff --git a/tests/test-fanning-in-collectors.py b/tests/test-fanning-in-collectors.py index b5da8612..866c5a20 100644 --- a/tests/test-fanning-in-collectors.py +++ b/tests/test-fanning-in-collectors.py @@ -1,7 +1,7 @@ import asyncio from pathlib import Path from querent.collectors.fs.fs_collector import FSCollectorFactory -from querent.config.collector_config import FSCollectorConfig +from querent.config.collector.collector_config import FSCollectorConfig from querent.common.uri import Uri from querent.ingestors.ingestor_manager import IngestorFactoryManager import pytest diff --git a/tests/test_async_collectors_ingestion.py b/tests/test_async_collectors_ingestion.py index 7261b2db..a7fb1db0 100644 --- a/tests/test_async_collectors_ingestion.py +++ b/tests/test_async_collectors_ingestion.py @@ -2,7 +2,7 @@ from asyncio import Queue from pathlib import Path from querent.collectors.fs.fs_collector import FSCollectorFactory -from querent.config.collector_config import FSCollectorConfig +from querent.config.collector.collector_config import FSCollectorConfig from querent.common.uri import Uri from querent.ingestors.ingestor_manager import IngestorFactoryManager import pytest diff --git a/tests/test_audio_ingestor.py b/tests/test_audio_ingestor.py index 48cfa779..1959f697 100644 --- a/tests/test_audio_ingestor.py +++ b/tests/test_audio_ingestor.py @@ -4,7 +4,7 @@ import asyncio from querent.collectors.fs.fs_collector import FSCollectorFactory -from querent.config.collector_config import FSCollectorConfig +from querent.config.collector.collector_config import FSCollectorConfig from querent.common.uri import Uri from querent.ingestors.ingestor_manager import IngestorFactoryManager import uuid diff --git a/tests/test_aws_collector.py b/tests/test_aws_collector.py index 17f596dd..32403013 100644 --- a/tests/test_aws_collector.py +++ b/tests/test_aws_collector.py @@ -1,10 +1,10 @@ import asyncio -from querent.config.collector_config import S3CollectConfig +from querent.config.collector.collector_config import S3CollectConfig from querent.collectors.collector_resolver import CollectorResolver from querent.collectors.aws.aws_collector import AWSCollectorFactory from querent.common.uri import Uri -from querent.config.collector_config import CollectorBackend +from querent.config.collector.collector_config import CollectorBackend import pytest import os from dotenv import load_dotenv diff --git a/tests/test_code_ingestor.py b/tests/test_code_ingestor.py index 1024194a..985aa989 100644 --- a/tests/test_code_ingestor.py +++ b/tests/test_code_ingestor.py @@ -1,7 +1,7 @@ import asyncio from pathlib import Path from querent.collectors.fs.fs_collector import FSCollectorFactory -from querent.config.collector_config import FSCollectorConfig +from querent.config.collector.collector_config import FSCollectorConfig from querent.common.uri import Uri from querent.ingestors.ingestor_manager import IngestorFactoryManager import pytest diff --git a/tests/test_csv_ingestor.py b/tests/test_csv_ingestor.py index 66668391..75396ead 100644 --- a/tests/test_csv_ingestor.py +++ b/tests/test_csv_ingestor.py @@ -3,7 +3,7 @@ from pathlib import Path from querent.collectors.fs.fs_collector import FSCollectorFactory -from querent.config.collector_config import FSCollectorConfig +from querent.config.collector.collector_config import FSCollectorConfig from querent.common.uri import Uri from querent.ingestors.ingestor_manager import IngestorFactoryManager import uuid diff --git a/tests/test_doc_ingestor.py b/tests/test_doc_ingestor.py index c481df13..428fd2e2 100644 --- a/tests/test_doc_ingestor.py +++ b/tests/test_doc_ingestor.py @@ -4,7 +4,7 @@ import asyncio from querent.collectors.fs.fs_collector import FSCollectorFactory -from querent.config.collector_config import FSCollectorConfig +from querent.config.collector.collector_config import FSCollectorConfig from querent.common.uri import Uri from querent.ingestors.ingestor_manager import IngestorFactoryManager import uuid diff --git a/tests/test_drive_collector.py b/tests/test_drive_collector.py index 0d3858f7..b9f57a04 100644 --- a/tests/test_drive_collector.py +++ b/tests/test_drive_collector.py @@ -2,7 +2,7 @@ import pytest import os from querent.collectors.collector_resolver import CollectorResolver -from querent.config.collector_config import DriveCollectorConfig +from querent.config.collector.collector_config import DriveCollectorConfig from querent.common.uri import Uri import uuid from dotenv import load_dotenv diff --git a/tests/test_dropbox_collector.py b/tests/test_dropbox_collector.py index 9c088427..ec7d98a7 100644 --- a/tests/test_dropbox_collector.py +++ b/tests/test_dropbox_collector.py @@ -2,7 +2,7 @@ import pytest import os from querent.collectors.collector_resolver import CollectorResolver -from querent.config.collector_config import DropboxConfig +from querent.config.collector.collector_config import DropboxConfig from querent.common.uri import Uri import uuid diff --git a/tests/test_email_collector.py b/tests/test_email_collector.py index ac7bee37..2811f91c 100644 --- a/tests/test_email_collector.py +++ b/tests/test_email_collector.py @@ -2,7 +2,7 @@ import pytest import os from querent.collectors.collector_resolver import CollectorResolver -from querent.config.collector_config import CollectorBackend, EmailCollectorConfig +from querent.config.collector.collector_config import CollectorBackend, EmailCollectorConfig from querent.common.uri import Uri import uuid from dotenv import load_dotenv diff --git a/tests/test_gcs_collector.py b/tests/test_gcs_collector.py index a5fb5425..79836f46 100644 --- a/tests/test_gcs_collector.py +++ b/tests/test_gcs_collector.py @@ -3,7 +3,7 @@ from querent.collectors.collector_resolver import CollectorResolver from querent.collectors.gcs.gcs_collector import GCSCollectorFactory from querent.common.uri import Uri -from querent.config.collector_config import CollectorBackend, GcsCollectConfig +from querent.config.collector.collector_config import CollectorBackend, GcsCollectConfig import pytest import uuid from dotenv import load_dotenv diff --git a/tests/test_generic_ingestor.py b/tests/test_generic_ingestor.py index d198fd9c..d7c5b02d 100644 --- a/tests/test_generic_ingestor.py +++ b/tests/test_generic_ingestor.py @@ -2,7 +2,7 @@ import os from pathlib import Path from querent.collectors.slack.slack_collector import SlackCollectorFactory -from querent.config.collector_config import SlackCollectorConfig +from querent.config.collector.collector_config import SlackCollectorConfig from querent.common.uri import Uri from querent.ingestors.ingestor_manager import IngestorFactoryManager import pytest diff --git a/tests/test_github_ingestor.py b/tests/test_github_ingestor.py index f908b19a..4b32b86c 100644 --- a/tests/test_github_ingestor.py +++ b/tests/test_github_ingestor.py @@ -1,7 +1,7 @@ import asyncio from pathlib import Path from querent.collectors.github.github_collector import GithubCollectorFactory -from querent.config.collector_config import GithubConfig +from querent.config.collector.collector_config import GithubConfig from querent.common.uri import Uri from querent.ingestors.ingestor_manager import IngestorFactoryManager import pytest diff --git a/tests/test_html_ingestor.py b/tests/test_html_ingestor.py index 86943311..4c7d04a0 100644 --- a/tests/test_html_ingestor.py +++ b/tests/test_html_ingestor.py @@ -1,7 +1,7 @@ import asyncio from pathlib import Path from querent.collectors.fs.fs_collector import FSCollectorFactory -from querent.config.collector_config import FSCollectorConfig +from querent.config.collector.collector_config import FSCollectorConfig from querent.common.uri import Uri from querent.ingestors.ingestor_manager import IngestorFactoryManager import pytest diff --git a/tests/test_image_ingestor.py b/tests/test_image_ingestor.py index 9c41d0b4..9a2cce9d 100644 --- a/tests/test_image_ingestor.py +++ b/tests/test_image_ingestor.py @@ -1,7 +1,7 @@ import asyncio from pathlib import Path from querent.collectors.fs.fs_collector import FSCollectorFactory -from querent.config.collector_config import FSCollectorConfig +from querent.config.collector.collector_config import FSCollectorConfig from querent.common.uri import Uri from querent.ingestors.ingestor_manager import IngestorFactoryManager import pytest diff --git a/tests/test_jira_collector.py b/tests/test_jira_collector.py index 2190ac6e..1e2dd2af 100644 --- a/tests/test_jira_collector.py +++ b/tests/test_jira_collector.py @@ -1,7 +1,7 @@ import asyncio import pytest from querent.collectors.collector_resolver import CollectorResolver -from querent.config.collector_config import JiraCollectorConfig +from querent.config.collector.collector_config import JiraCollectorConfig from querent.common.uri import Uri import uuid from dotenv import load_dotenv diff --git a/tests/test_json_ingestor.py b/tests/test_json_ingestor.py index f7529fd1..427c1075 100644 --- a/tests/test_json_ingestor.py +++ b/tests/test_json_ingestor.py @@ -1,7 +1,7 @@ import asyncio from pathlib import Path from querent.collectors.fs.fs_collector import FSCollectorFactory -from querent.config.collector_config import FSCollectorConfig +from querent.config.collector.collector_config import FSCollectorConfig from querent.common.uri import Uri from querent.ingestors.ingestor_manager import IngestorFactoryManager import pytest diff --git a/tests/test_pdf_ingestor.py b/tests/test_pdf_ingestor.py index 0051f514..cd3610ea 100644 --- a/tests/test_pdf_ingestor.py +++ b/tests/test_pdf_ingestor.py @@ -1,7 +1,7 @@ import asyncio from pathlib import Path from querent.collectors.fs.fs_collector import FSCollectorFactory -from querent.config.collector_config import FSCollectorConfig +from querent.config.collector.collector_config import FSCollectorConfig from querent.common.uri import Uri from querent.ingestors.ingestor_manager import IngestorFactoryManager import pytest diff --git a/tests/test_ppt_ingestor.py b/tests/test_ppt_ingestor.py index e2634cd7..f9f0fc8c 100644 --- a/tests/test_ppt_ingestor.py +++ b/tests/test_ppt_ingestor.py @@ -1,7 +1,7 @@ import asyncio from pathlib import Path from querent.collectors.fs.fs_collector import FSCollectorFactory -from querent.config.collector_config import FSCollectorConfig +from querent.config.collector.collector_config import FSCollectorConfig from querent.common.uri import Uri from querent.ingestors.ingestor_manager import IngestorFactoryManager import pytest diff --git a/tests/test_text_ingestor.py b/tests/test_text_ingestor.py index a00595cf..7cb7f9e4 100644 --- a/tests/test_text_ingestor.py +++ b/tests/test_text_ingestor.py @@ -1,7 +1,7 @@ import asyncio from pathlib import Path from querent.collectors.fs.fs_collector import FSCollectorFactory -from querent.config.collector_config import FSCollectorConfig +from querent.config.collector.collector_config import FSCollectorConfig from querent.common.uri import Uri from querent.ingestors.ingestor_manager import IngestorFactoryManager import pytest diff --git a/tests/test_webscrapper.py b/tests/test_webscrapper.py index 28be528e..3035cd1b 100644 --- a/tests/test_webscrapper.py +++ b/tests/test_webscrapper.py @@ -7,7 +7,7 @@ import uuid from querent.common.uri import Uri -from querent.config.collector_config import CollectorBackend, WebScraperConfig +from querent.config.collector.collector_config import CollectorBackend, WebScraperConfig def test_webscrapper_collector(): diff --git a/tests/test_xlsx_ingestor.py b/tests/test_xlsx_ingestor.py index 029cc8c9..cf998f88 100644 --- a/tests/test_xlsx_ingestor.py +++ b/tests/test_xlsx_ingestor.py @@ -2,7 +2,7 @@ from pathlib import Path from querent.collectors.fs.fs_collector import FSCollectorFactory from querent.common.types.ingested_tokens import IngestedTokens -from querent.config.collector_config import FSCollectorConfig +from querent.config.collector.collector_config import FSCollectorConfig from querent.common.uri import Uri from querent.ingestors.ingestor_manager import IngestorFactoryManager import pytest diff --git a/tests/test_xml_ingestor.py b/tests/test_xml_ingestor.py index ebc27521..407e3d08 100644 --- a/tests/test_xml_ingestor.py +++ b/tests/test_xml_ingestor.py @@ -1,7 +1,7 @@ import asyncio from pathlib import Path from querent.collectors.fs.fs_collector import FSCollectorFactory -from querent.config.collector_config import FSCollectorConfig +from querent.config.collector.collector_config import FSCollectorConfig from querent.common.uri import Uri from querent.ingestors.ingestor_manager import IngestorFactoryManager import pytest