Merge pull request #1 from TogetherCrew/WIP-service-setup

Service has fully set up!

.env.example

@@ -0,0 +1 @@
+GITHUB_TOKEN=daolytics_access_token

.gitignore

@@ -167,7 +167,7 @@ cython_debug/
 # Emacs
 .org
 
-interactions/credentials.py
-interactions/temp.ipynb
-test*.ipynb
-analyzer/rndao_analyzer/analysis/credentials.py
+coverage/*
+
+analyzer_lib
+main.ipynb

Dockerfile

@@ -1,12 +1,16 @@
 # It's recommended that we use `bullseye` for Python (alpine isn't suitable as it conflcts with numpy)
-FROM python:3.10-bullseye AS base 
+FROM python:3.10-bullseye AS base
 WORKDIR /project
 COPY . .
+ARG GITHUB_TOKEN
 RUN pip3 install -r requirements.txt
 
 FROM base AS test
-RUN python3 -m coverage run -m pytest tests
-CMD ["python3", "-m", "coverage", "lcov" ,"-o", "coverage/lcov.info"]
+RUN chmod +x docker-entrypoint.sh
+CMD ["./docker-entrypoint.sh"]
 
-FROM base AS prod
-CMD ["python3", "server.py"]
+FROM base AS prod-server
+CMD ["python3", "start_rabbit_mq.py"]
+
+FROM base as prod-worker
+CMD ["python3", "redis_worker.py"]

README.md

@@ -1 +1,6 @@
-# python-service
+# Discord-Analyzer
+
+[![Maintainability](https://api.codeclimate.com/v1/badges/e1239b895f0ee2569b61/maintainability)](https://codeclimate.com/github/TogetherCrew/discord-analyzer/maintainability)
+[![Test Coverage](https://api.codeclimate.com/v1/badges/e1239b895f0ee2569b61/test_coverage)](https://codeclimate.com/github/TogetherCrew/discord-analyzer/test_coverage)
+
+This repository contains the codes to analyze discord chat data.

analyzer_init.py

@@ -0,0 +1,57 @@
+from typing import Any
+
+from discord_analyzer import RnDaoAnalyzer
+from utils.daolytics_uitls import (
+    get_mongo_credentials,
+    get_neo4j_credentials,
+    get_saga_db_location,
+)
+
+
+class AnalyzerInit:
+    """
+    initialize the analyzer with its configs
+    """
+
+    def __init__(self) -> None:
+        pass
+
+    def get_analyzer(self) -> tuple[RnDaoAnalyzer, dict[str, Any]]:
+        """
+        Returns:
+        ---------
+        analyzer : RnDaoAnalyzer
+        mongo_creds : dict[str, Any]
+        """
+        analyzer = RnDaoAnalyzer()
+
+        # credentials
+        mongo_creds = get_mongo_credentials()
+        neo4j_creds = get_neo4j_credentials()
+        saga_mongo_location = get_saga_db_location()
+
+        mongo_creds["db_name"] = saga_mongo_location["db_name"]
+        mongo_creds["collection_name"] = saga_mongo_location["collection_name"]
+        mongo_creds["connection_str"] = self._get_mongo_connection(mongo_creds)
+
+        analyzer.set_mongo_database_info(
+            mongo_db_host=mongo_creds["host"],
+            mongo_db_password=mongo_creds["password"],
+            mongo_db_port=mongo_creds["port"],
+            mongo_db_user=mongo_creds["user"],
+        )
+        analyzer.set_neo4j_database_info(neo4j_creds=neo4j_creds)
+        analyzer.database_connect()
+        analyzer.setup_neo4j_metrics()
+
+        return analyzer, mongo_creds
+
+    def _get_mongo_connection(self, mongo_creds: dict[str, Any]):
+        user = mongo_creds["user"]
+        password = mongo_creds["password"]
+        host = mongo_creds["host"]
+        port = mongo_creds["port"]
+
+        connection = f"mongodb://{user}:{password}@{host}:{port}"
+
+        return connection

discord_analyzer/DB_operations/mongo_neo4j_ops.py

@@ -0,0 +1,171 @@
+import logging
+
+from discord_analyzer.DB_operations.mongodb_interaction import MongoDBOps
+from discord_analyzer.DB_operations.network_graph import make_neo4j_networkx_query_dict
+from tc_neo4j_lib.neo4j_ops import Neo4jOps
+
+
+class MongoNeo4jDB:
+    def __init__(self, testing=False):
+        """
+        having both databases in one class
+
+        """
+        self.neo4j_ops = None
+        self.mongoOps = None
+        self.testing = testing
+
+    def set_neo4j_utils(
+        self,
+        db_name: str,
+        host: str,
+        port: str,
+        protocol: str,
+        user: str,
+        password: str,
+    ):
+        """
+        store the neo4j utils instance
+        """
+        self.neo4j_ops = Neo4jOps()
+        self.neo4j_ops.set_neo4j_db_info(
+            neo4j_db_name=db_name,
+            neo4j_protocol=protocol,
+            neo4j_user=user,
+            neo4j_password=password,
+            neo4j_host=host,
+            neo4j_port=port,
+        )
+        self.neo4j_ops.neo4j_database_connect()
+
+    def set_mongo_db_ops(
+        self, mongo_user: str, mongo_pass: str, mongo_host: str, mongo_port: str
+    ):
+        """
+        setup the MongoDBOps class with the parameters needed
+
+        """
+        self.mongoOps = MongoDBOps(
+            user=mongo_user, password=mongo_pass, host=mongo_host, port=mongo_port
+        )
+        self.mongoOps.set_mongo_db_access()
+
+    def store_analytics_data(
+        self, analytics_data, remove_memberactivities=False, remove_heatmaps=False
+    ):
+        """
+        store the analytics data into database
+        all data are in format of nested dictionaries which
+
+        Parameters:
+        -------------
+        analytics_data : dictionary
+            a nested dictinoary with keys as guildId
+            and values as heatmaps and memberactivities data
+            heatmaps is also a list of dictinoaries
+            and memberactivities is a tuple of memberactivities dictionary list
+             and memebractivities networkx object dictionary list
+        remove_memberactivities : bool
+            remove the whole memberactivity data and insert
+            default is `False` which means don't delete the existing data
+        remove_heatmaps : bool
+            remove the whole heatmap data and insert
+            default is `False` which means don't delete the existing data
+
+        Returns:
+        ----------
+        `None`
+        """
+        for guildId in analytics_data.keys():
+            heatmaps_data = analytics_data[guildId]["heatmaps"]
+            (memberactivities_data, memberactivities_networkx_data) = analytics_data[
+                guildId
+            ]["memberactivities"]
+
+            if not self.testing:
+                # mongodb transactions
+                self.mongoOps._do_analytics_write_transaction(
+                    guildId=guildId,
+                    delete_heatmaps=remove_heatmaps,
+                    delete_member_acitivities=remove_memberactivities,
+                    acitivties_list=memberactivities_data,
+                    heatmaps_list=heatmaps_data,
+                )
+
+                # neo4j transactions
+                if (
+                    memberactivities_networkx_data is not None
+                    and memberactivities_networkx_data != []
+                ):
+                    queries_list = make_neo4j_networkx_query_dict(
+                        networkx_graphs=memberactivities_networkx_data, guildId=guildId
+                    )
+                    self.run_operations_transaction(
+                        guildId=guildId,
+                        queries_list=queries_list,
+                        remove_memberactivities=remove_memberactivities,
+                    )
+            else:
+                logging.warning("Testing mode enabled! Not saving any data")
+
+    def run_operations_transaction(
+        self, guildId, queries_list, remove_memberactivities
+    ):
+        """
+        do the deletion and insertion operations inside a transaction
+
+        Parameters:
+        ------------
+        guildId : str
+            the guild id that the users are connected to it
+            which we're going to delete the relations of it
+        queries_list : list
+            list of strings to add data into neo4j
+            min length is 1
+        remove_memberactivities : bool
+            if True, remove the old data specified in that guild
+        """
+        self.guild_msg = f"GUILDID: {guildId}:"
+
+        transaction_queries = []
+        if remove_memberactivities:
+            logging.info(
+                f"{self.guild_msg} Neo4J GuildId accounts relation will be removed!"
+            )
+            delete_relationship_query = self._create_guild_rel_deletion_query(
+                guildId=guildId
+            )
+            transaction_queries.append(delete_relationship_query)
+
+        # logging.info(queries_list)
+        transaction_queries.extend(queries_list)
+
+        self.neo4j_ops.store_data_neo4j(transaction_queries, message=self.guild_msg)
+
+    def _create_guild_rel_deletion_query(
+        self, guildId: str, relation_name: str = "INTERACTED_WITH"
+    ):
+        """
+        create a query to delete the relationships
+        between DiscordAccount users in a specific guild
+
+        Parameters:
+        -------------
+        guildId : str
+            the guild id that the users are connected to it
+        relation_name : str
+            the relation we want to delete
+
+        Returns:
+        ------------
+        final_query : str
+            the final query to remove the relationships
+        """
+
+        delete_relationship_query = f"""
+          MATCH
+            (:DiscordAccount)
+                -[r:{relation_name} {{guildId: '{guildId}'}}]-(:DiscordAccount)
+            DETACH DELETE r"""
+
+        return delete_relationship_query