added discord connector

SearchProviders/discord.json

@@ -0,0 +1,23 @@
+{
+    "name": "Discord",
+    "active": true,
+    "default": true,
+    "connector": "Discord",
+    "url": "",
+    "query_template": "Fetch last {limit} messages from channel {channel_id}",
+    "query_processors": [
+        "GenericQueryProcessor"
+    ],
+    "query_mappings": "NOT=True",
+    "result_processors": [
+        "MappingResultProcessor",
+        "CosineRelevancyResultProcessor"
+    ],
+    "response_mappings": "",
+    "result_mappings": "",
+    "results_per_query": 20,
+    "credentials": "add token here",
+    "tags": [
+        "Discord"
+    ]
+}

SearchProviders/qdrant.json

@@ -0,0 +1,23 @@
+{
+    "name": "qdrant",
+    "active": true,
+    "default": true,
+    "connector": "Qdrant",
+    "url": "",
+    "query_template": "",
+    "query_processors": [
+        "GenericQueryProcessor"
+    ],
+    "query_mappings": "",
+    "result_processors": [
+        "MappingResultProcessor",
+        "CosineRelevancyResultProcessor"
+    ],
+    "response_mappings": "",
+    "result_mappings": "",
+    "results_per_query": 20,
+    "credentials": "",
+    "tags": [
+        "Qdrant"
+    ]
+}

requirements.txt

@@ -29,4 +29,7 @@ readability-lxml
 tiktoken
 channels
 channels-redis
-tika
+tika
+discord.py
+cohere
+qdrant-client

swirl/connectors/__init__.py

@@ -15,7 +15,8 @@
 from swirl.connectors.microsoft_graph import M365OutlookCalendar
 from swirl.connectors.microsoft_graph import M365SharePointSites
 from swirl.connectors.microsoft_graph import MicrosoftTeams
-
+from swirl.connectors.discord import Discord
+from swirl.connectors.qdrant import Qdrant
 
 # uncomment the line below to enable PostgreSQL
 # from swirl.connectors.postgresql import PostgreSQL

swirl/connectors/discord.py

@@ -0,0 +1,63 @@
+import asyncio
+
+import discord
+from django.conf import settings
+
+from swirl.connectors.connector import Connector
+from swirl.connectors.mappings import *
+logger = get_task_logger(__name__)
+path.append(swirl_setdir())
+environ.setdefault('DJANGO_SETTINGS_MODULE', 'swirl_server.settings')
+django.setup()
+
+
+class Discord(Connector):
+    type = "Discord"
+
+    def __init__(self, provider_id, search_id, update, request_id=''):
+        self.client = discord.Client(intents=discord.Intents.all())
+        super().__init__(provider_id, search_id, update, request_id)
+        self.client.event(self.on_ready)
+
+    async def on_ready(self):
+        await self.client.wait_until_ready()
+        await asyncio.sleep(2)
+        channel_id = 718114919056146528
+        target_channel = self.client.get_channel(channel_id)
+        if target_channel is None:
+            logger.error(f"Channel {channel_id} not found")
+            self.status = "ERR_CHANNEL_NOT_FOUND"
+            return
+
+        messages_data = []
+        channel_name = target_channel.name
+        async for message in target_channel.history(limit=int(self.provider.results_per_query)):
+            messages_data.append({
+                "author": message.author.name,
+                "body": message.content,
+                "title": f"{message.author.name}_{channel_name}_{message.created_at}"
+            })
+        self.response = messages_data
+        logger.debug(f"data={messages_data}")
+        await self.client.close()
+
+    def execute_search(self, session=None):
+        if self.provider.credentials:
+            token = self.provider.credentials
+        else:
+            if getattr(settings, 'DISCORD_BOT_TOKEN', None):
+                token = settings.DISCORD_BOT_TOKEN
+            else:
+                self.status = "ERR_NO_CREDENTIALS"
+                return
+        try:
+            self.client.run(token)
+        except Exception as e:
+            logger.error(f"Failed to start client: {e}")
+            self.status = "ERR_CLIENT_START_FAILURE"
+
+    def normalize_response(self):
+
+        logger.debug(f"{self}: normalize_response()")
+        self.results = self.response
+        return

swirl/connectors/qdrant.py

@@ -0,0 +1,82 @@
+import asyncio
+
+from django.conf import settings
+import cohere
+from qdrant_client import QdrantClient, models
+from qdrant_client.http.models import Batch
+from swirl.connectors.connector import Connector
+from swirl.connectors.mappings import *
+from swirl.connectors.utils import bind_query_mappings
+
+logger = get_task_logger(__name__)
+path.append(swirl_setdir())
+environ.setdefault('DJANGO_SETTINGS_MODULE', 'swirl_server.settings')
+django.setup()
+
+
+class Qdrant(Connector):
+    type = "Qdrant"
+
+    def __init__(self, provider_id, search_id, update, request_id='', resumes=None, jd=None):
+        super().__init__(provider_id, search_id, update, request_id)
+        self.cohere_key = getattr(settings, 'COHERE_API_KEY', None)
+        self.qdrant_key = getattr(settings, 'QDRANT_API_KEY', None)
+        self.qdrant_url = getattr(settings, 'QDRANT_URL', None)
+        self.cohere = cohere.Client(self.cohere_key)
+        self.collection_name = "your_collection_name"
+        self.qdrant = QdrantClient(
+            url=self.qdrant_url,
+            api_key=self.qdrant_key,
+        )
+
+    def get_embedding(self, text):
+        try:
+            embeddings = self.cohere.embed([text], "large").embeddings
+            return list(map(float, embeddings[0])), len(embeddings[0])
+        except Exception as e:
+            logger.error(f"Error getting embeddings: {e}", exc_info=True)
+
+    def get_result(self):
+        vector, _ = self.get_embedding(self.query_to_provider)
+
+        hits = self.qdrant.search(
+            collection_name=self.collection_name,
+            query_vector=vector,
+            limit=30
+        )
+        results = []
+        for hit in hits:
+            result = {
+                'text': str(hit.payload),
+                'score': hit.score
+            }
+            results.append(result)
+        logger.debug(f"qdrant results={results}")
+        final_text = []
+        for result in results:
+            final_text.append(result["text"][10:])
+        return final_text
+
+    def construct_query(self):
+        logger.debug(f"{self}: construct_query()")
+
+        self.query_to_provider = self.query_string_to_provider
+        logger.info(f"sending query: {self.query_to_provider}")
+
+        return
+
+    def execute_search(self, session=None):
+        logger.debug("Started")
+        data = self.get_result()
+        final_data = [{
+            "body": data,
+            "title": f"Qdrant_search"
+        }]
+        logger.debug(f"final data={final_data}")
+        self.response = final_data
+        logger.info("Finished getting response")
+
+    def normalize_response(self):
+        logger.debug(f"{self}: normalize_response()")
+        self.results = self.response
+        return

swirl/models.py

@@ -67,6 +67,8 @@ class SearchProvider(models.Model):
         'MicrosoftTeams': 'Microsoft',
     })
     CONNECTOR_CHOICES = [
+        ('Qdrant', 'Qdrant'),
+        ('Discord', 'Discord'),
         ('ChatGPT', 'ChatGPT Query String'),
         ('RequestsGet', 'HTTP/GET returning JSON'),
         ('RequestsPost', 'HTTP/POST returning JSON'),

swirl_server/settings.py

@@ -286,4 +286,9 @@
         },
     },
 }
-ASGI_THREADS = 1000
+ASGI_THREADS = 1000
+
+DISCORD_BOT_TOKEN = env("DISCORD_BOT_TOKEN")
+COHERE_API_KEY = env("COHERE_API_KEY")
+QDRANT_API_KEY = env("QDRANT_API_KEY")
+QDRANT_URL = env("QDRANT_URL")