reorganize configs, cleanups and add a workflow config

querent/collectors/aws/aws_collector.py

@@ -6,7 +6,7 @@
 from querent.common.types.collected_bytes import (
     CollectedBytes,
 )  # Import for handling authentication errors
-from querent.config.collector_config import CollectorBackend, S3CollectConfig
+from querent.config.collector.collector_config import CollectorBackend, S3CollectConfig
 from querent.collectors.collector_base import Collector
 from querent.collectors.collector_factory import CollectorFactory
 from querent.common.uri import Uri

querent/collectors/azure/azure_collector.py

@@ -3,7 +3,7 @@
 
 from azure.storage.blob import BlobServiceClient
 from querent.common.types.collected_bytes import CollectedBytes
-from querent.config.collector_config import CollectorBackend, AzureCollectConfig
+from querent.config.collector.collector_config import CollectorBackend, AzureCollectConfig
 from querent.collectors.collector_base import Collector
 from querent.collectors.collector_factory import CollectorFactory
 from querent.common.uri import Uri

querent/collectors/collector_factory.py

@@ -6,7 +6,7 @@
     CollectorResolverError,
     CollectorErrorKind,
 )
-from querent.config.collector_config import CollectorConfig, CollectorBackend
+from querent.config.collector.collector_config import CollectorConfig, CollectorBackend
 
 
 class CollectorFactory(ABC):

querent/collectors/collector_resolver.py

@@ -10,7 +10,7 @@
 from querent.collectors.dropbox.dropbox_collector import DropBoxCollectorFactory
 from querent.collectors.github.github_collector import GithubCollectorFactory
 from querent.collectors.drive.google_drive_collector import DriveCollectorFactory
-from querent.config.collector_config import CollectorConfig, CollectorBackend
+from querent.config.collector.collector_config import CollectorConfig, CollectorBackend
 from querent.collectors.collector_base import Collector
 from querent.collectors.collector_errors import (
     CollectorResolverError,

querent/collectors/drive/google_drive_collector.py

@@ -12,7 +12,7 @@
 from querent.collectors.collector_factory import CollectorFactory
 from querent.common.types.collected_bytes import CollectedBytes
 from querent.common.uri import Uri
-from querent.config.collector_config import CollectorBackend, DriveCollectorConfig
+from querent.config.collector.collector_config import CollectorBackend, DriveCollectorConfig
 from querent.common import common_errors
 import requests
 

querent/collectors/dropbox/dropbox_collector.py

@@ -3,9 +3,9 @@
 import dropbox
 from dropbox.oauth import DropboxOAuth2FlowNoRedirect
 from querent.common.types.collected_bytes import CollectedBytes
-from querent.config.collector_config import DropboxConfig
+from querent.config.collector.collector_config import DropboxConfig
 from querent.collectors.collector_base import Collector
-from querent.config.collector_config import CollectorBackend
+from querent.config.collector.collector_config import CollectorBackend
 from querent.collectors.collector_factory import CollectorFactory
 from querent.common import common_errors
 from querent.common.uri import Uri

querent/collectors/email/email_collector.py

@@ -8,7 +8,7 @@
 from querent.common import common_errors
 from querent.common.types.collected_bytes import CollectedBytes
 from querent.common.uri import Uri
-from querent.config.collector_config import CollectorBackend, EmailCollectorConfig
+from querent.config.collector.collector_config import CollectorBackend, EmailCollectorConfig
 from querent.logging.logger import setup_logger
 
 

querent/collectors/email/imap.py

@@ -1,6 +1,6 @@
 import imaplib
 
-from querent.config.collector_config import EmailCollectorConfig
+from querent.config.collector.collector_config import EmailCollectorConfig
 
 
 class ImapEmail:

querent/collectors/fs/fs_collector.py

@@ -7,7 +7,7 @@
 from querent.collectors.collector_factory import CollectorFactory
 from querent.common.types.collected_bytes import CollectedBytes
 from querent.common.uri import Uri
-from querent.config.collector_config import CollectorBackend, FSCollectorConfig
+from querent.config.collector.collector_config import CollectorBackend, FSCollectorConfig
 import aiofiles
 from querent.common import common_errors
 

querent/collectors/gcs/gcs_collector.py

@@ -3,8 +3,8 @@
 
 import aiofiles
 from querent.common.types.collected_bytes import CollectedBytes
-from querent.config.collector_config import GcsCollectConfig
-from querent.config.collector_config import CollectorBackend
+from querent.config.collector.collector_config import GcsCollectConfig
+from querent.config.collector.collector_config import CollectorBackend
 from querent.collectors.collector_base import Collector
 from querent.collectors.collector_factory import CollectorFactory
 from querent.common.uri import Uri

querent/collectors/github/github_collector.py

@@ -3,7 +3,7 @@
 from querent.common.types.collected_bytes import CollectedBytes
 
 
-from querent.config.collector_config import GithubConfig, CollectorBackend
+from querent.config.collector.collector_config import GithubConfig, CollectorBackend
 from querent.collectors.collector_base import Collector
 from querent.collectors.collector_factory import CollectorFactory
 from querent.common.uri import Uri

querent/collectors/jira/jira_collector.py

@@ -6,7 +6,7 @@
 from querent.common import common_errors
 from querent.common.types.collected_bytes import CollectedBytes
 from querent.common.uri import Uri
-from querent.config.collector_config import CollectorBackend, JiraCollectorConfig
+from querent.config.collector.collector_config import CollectorBackend, JiraCollectorConfig
 from jira import JIRA
 
 from querent.logging.logger import setup_logger

querent/collectors/slack/slack_collector.py

@@ -4,7 +4,7 @@
 import os
 
 from querent.collectors.collector_factory import CollectorFactory
-from querent.config.collector_config import CollectorBackend, SlackCollectorConfig
+from querent.config.collector.collector_config import CollectorBackend, SlackCollectorConfig
 from querent.collectors.collector_base import Collector
 from querent.common.types.collected_bytes import CollectedBytes
 from querent.common.uri import Uri

querent/collectors/webscaper/web_scraper_collector.py

@@ -3,7 +3,7 @@
 from querent.collectors.collector_base import Collector
 from querent.collectors.collector_factory import CollectorFactory
 from querent.common.types.collected_bytes import CollectedBytes
-from querent.config.collector_config import CollectorBackend, WebScraperConfig
+from querent.config.collector.collector_config import CollectorBackend, WebScraperConfig
 from querent.common.uri import Uri
 from querent.tools.web_page_extractor import WebpageExtractor
 from urllib.parse import urlparse, urljoin

querent/config/collector_config.py → querent/config/collector/collector_config.py

@@ -1,6 +1,6 @@
 from enum import Enum
-from typing import List, Optional, Union
-from pydantic import BaseModel, Field
+from typing import Any, List, Optional, Union
+from pydantic import BaseModel, Field, validator
 
 from querent.channel.channel_interface import ChannelCommandInterface
 
@@ -21,7 +21,14 @@ class CollectorBackend(str, Enum):
 
 class CollectorConfig(BaseModel):
     backend: CollectorBackend
-    channel: ChannelCommandInterface
+    # Use Field with allow_mutation=False to specify the type
+    channel: Any
+
+    # Custom validator for ChannelCommandInterface
+    @validator("channel", pre=True, allow_reuse=True)
+    def validate_channel(cls, value):
+        # Perform any additional validation logic here
+        return value
 
 
 class FSCollectorConfig(CollectorConfig):

querent/config/engine_config.py → querent/config/engine/engine_config.py

@@ -1,4 +1,5 @@
-from pydantic import BaseModel
+from typing import Any
+from pydantic import BaseModel, Field, validator
 
 from querent.channel.channel_interface import ChannelCommandInterface
 
@@ -15,7 +16,14 @@ class EngineConfig(BaseModel):
     retry_interval: float = 2.0
     message_throttle_limit: int = 1000
     message_throttle_delay: float = 0.001
-    channel: ChannelCommandInterface
+    # Use Field with allow_mutation=False to specify the type
+    channel: Any
     logger: str = f"{__name__}.engine_config"
     state_queue: str = f"{__name__}.state_queue"
     workers: str = f"{__name__}.workers"
+
+    # Custom validator for ChannelCommandInterface
+    @validator("channel", pre=True, allow_reuse=True)
+    def validate_channel(cls, value):
+        # Perform any additional validation logic here
+        return value

querent/config/workflow/workflow_config.py

@@ -0,0 +1,21 @@
+from typing import Any
+from pydantic import BaseModel, validator
+
+
+class WorkflowConfig(BaseModel):
+    """Workflow configuration."""
+
+    name: str
+    id: str
+    channel: Any
+    event_handler: Any
+
+    @validator("channel", pre=True, allow_reuse=True)
+    def validate_channel(cls, value):
+        # Perform any additional validation logic here
+        return value
+
+    @validator("event_handler", pre=True, allow_reuse=True)
+    def validate_event_handler(cls, value):
+        # Perform any additional validation logic here
+        return value

querent/core/base_engine.py

@@ -7,7 +7,7 @@
 from querent.common.types.ingested_code import IngestedCode
 from querent.common.types.querent_event import EventState, EventType
 from querent.common.types.querent_queue import QuerentQueue
-from querent.config.engine_config import EngineConfig
+from querent.config.engine.engine_config import EngineConfig
 from querent.logging.logger import setup_logger
 
 """

querent/ingestors/audio/audio_ingestors.py

@@ -6,7 +6,7 @@
 from querent.ingestors.ingestor_factory import IngestorFactory
 from querent.processors.async_processor import AsyncProcessor
 from querent.ingestors.base_ingestor import BaseIngestor
-from querent.config.ingestor_config import IngestorBackend
+from querent.config.ingestor.ingestor_config import IngestorBackend
 from querent.common.types.collected_bytes import CollectedBytes
 from querent.common.common_errors import (
     UnknownValueError,

querent/ingestors/code/code_ingestor.py

@@ -3,7 +3,7 @@
 from querent.ingestors.base_ingestor import BaseIngestor
 from querent.ingestors.ingestor_factory import IngestorFactory
 from querent.processors.async_processor import AsyncProcessor
-from querent.config.ingestor_config import IngestorBackend
+from querent.config.ingestor.ingestor_config import IngestorBackend
 from querent.common import common_errors
 from querent.common.types.ingested_tokens import IngestedTokens
 

querent/ingestors/csv/csv_ingestor.py

@@ -5,7 +5,7 @@
 from querent.processors.async_processor import AsyncProcessor
 from querent.ingestors.ingestor_factory import IngestorFactory
 from querent.ingestors.base_ingestor import BaseIngestor
-from querent.config.ingestor_config import IngestorBackend
+from querent.config.ingestor.ingestor_config import IngestorBackend
 from querent.common.types.collected_bytes import CollectedBytes
 from querent.common import common_errors
 from querent.common.types.ingested_tokens import IngestedTokens

querent/ingestors/doc/doc_ingestor.py

@@ -7,7 +7,7 @@
 from querent.processors.async_processor import AsyncProcessor
 from querent.ingestors.ingestor_factory import IngestorFactory
 from querent.ingestors.base_ingestor import BaseIngestor
-from querent.config.ingestor_config import IngestorBackend
+from querent.config.ingestor.ingestor_config import IngestorBackend
 from querent.common.types.collected_bytes import CollectedBytes
 from querent.common import common_errors
 from querent.common.types.ingested_tokens import IngestedTokens

querent/ingestors/github/github_ingestor.py

@@ -3,7 +3,7 @@
 from querent.ingestors.base_ingestor import BaseIngestor
 from querent.ingestors.ingestor_factory import IngestorFactory
 from querent.processors.async_processor import AsyncProcessor
-from querent.config.ingestor_config import IngestorBackend
+from querent.config.ingestor.ingestor_config import IngestorBackend
 from querent.common.types.ingested_code import IngestedCode
 from querent.common.types.collected_bytes import CollectedBytes
 

querent/ingestors/html/html_ingestor.py

@@ -4,7 +4,7 @@
 from querent.processors.async_processor import AsyncProcessor
 from querent.ingestors.ingestor_factory import IngestorFactory
 from querent.ingestors.base_ingestor import BaseIngestor
-from querent.config.ingestor_config import IngestorBackend
+from querent.config.ingestor.ingestor_config import IngestorBackend
 from querent.common.types.collected_bytes import CollectedBytes
 from querent.common import common_errors
 from querent.common.types.ingested_tokens import IngestedTokens

querent/ingestors/images/image_ingestor.py

@@ -3,7 +3,7 @@
 from querent.ingestors.base_ingestor import BaseIngestor
 from querent.ingestors.ingestor_factory import IngestorFactory
 from querent.processors.async_processor import AsyncProcessor
-from querent.config.ingestor_config import IngestorBackend
+from querent.config.ingestor.ingestor_config import IngestorBackend
 from querent.processors.async_processor import AsyncProcessor
 from querent.common.common_errors import (
     FileNotFoundError,

querent/ingestors/ingestor_manager.py

@@ -6,7 +6,7 @@
 from querent.collectors.collector_base import Collector
 from querent.common.types.collected_bytes import CollectedBytes
 from querent.common.types.ingested_tokens import IngestedTokens
-from querent.config.ingestor_config import IngestorBackend
+from querent.config.ingestor.ingestor_config import IngestorBackend
 from querent.ingestors.base_ingestor import BaseIngestor
 from querent.ingestors.ingestor_factory import IngestorFactory, UnsupportedIngestor
 from querent.ingestors.pdfs.pdf_ingestor_v1 import PdfIngestorFactory

querent/ingestors/json/json_ingestor.py

@@ -1,7 +1,7 @@
 from typing import AsyncGenerator, List
 import json
 from querent.common.types.collected_bytes import CollectedBytes
-from querent.config.ingestor_config import IngestorBackend
+from querent.config.ingestor.ingestor_config import IngestorBackend
 from querent.ingestors.base_ingestor import BaseIngestor
 from querent.ingestors.ingestor_factory import IngestorFactory
 from querent.processors.async_processor import AsyncProcessor

querent/ingestors/pdfs/pdf_ingestor_v1.py

@@ -3,7 +3,7 @@
 from querent.common.types.collected_bytes import CollectedBytes
 from querent.common.types.ingested_tokens import IngestedTokens
 from querent.common.types.ingested_images import IngestedImages
-from querent.config.ingestor_config import IngestorBackend
+from querent.config.ingestor.ingestor_config import IngestorBackend
 from querent.ingestors.base_ingestor import BaseIngestor
 from querent.ingestors.ingestor_factory import IngestorFactory
 from querent.processors.async_processor import AsyncProcessor

querent/ingestors/ppt/ppt_ingestor.py

@@ -8,7 +8,7 @@
 from querent.ingestors.ingestor_factory import IngestorFactory
 from querent.processors.async_processor import AsyncProcessor
 from querent.ingestors.base_ingestor import BaseIngestor
-from querent.config.ingestor_config import IngestorBackend
+from querent.config.ingestor.ingestor_config import IngestorBackend
 from querent.common.types.collected_bytes import CollectedBytes
 from querent.common import common_errors
 from querent.common.types.ingested_tokens import IngestedTokens

querent/ingestors/texts/text_ingestor.py

@@ -3,7 +3,7 @@
 from querent.ingestors.base_ingestor import BaseIngestor
 from querent.ingestors.ingestor_factory import IngestorFactory
 from querent.processors.async_processor import AsyncProcessor
-from querent.config.ingestor_config import IngestorBackend
+from querent.config.ingestor.ingestor_config import IngestorBackend
 from querent.common import common_errors
 from querent.common.types.ingested_tokens import IngestedTokens
 

querent/ingestors/video/video_ingestor.py

@@ -4,7 +4,7 @@
 from querent.ingestors.ingestor_factory import IngestorFactory
 from querent.processors.async_processor import AsyncProcessor
 from querent.ingestors.base_ingestor import BaseIngestor
-from querent.config.ingestor_config import IngestorBackend
+from querent.config.ingestor.ingestor_config import IngestorBackend
 from querent.common.types.collected_bytes import CollectedBytes
 from querent.common.types.ingested_tokens import IngestedTokens
 import moviepy.editor as mp

querent/ingestors/xlsx/xlsx_ingestor.py

@@ -5,7 +5,7 @@
 from querent.ingestors.ingestor_factory import IngestorFactory
 from querent.ingestors.base_ingestor import BaseIngestor
 from querent.processors.async_processor import AsyncProcessor
-from querent.config.ingestor_config import IngestorBackend
+from querent.config.ingestor.ingestor_config import IngestorBackend
 from querent.common.types.collected_bytes import CollectedBytes
 from querent.common.types.ingested_tokens import (
     IngestedTokens,