Separate logger (#34)

* add motleycrew logger

* transferring logger from utils.py in logging.py

* replace logging calls with logger calls

---------

Co-authored-by: User <[email protected]>

examples/_test_single_llama_index.py

@@ -4,7 +4,7 @@
 
 from motleycrew import MotleyCrew
 from motleycrew.agents.llama_index import ReActLlamaIndexMotleyAgent
-from motleycrew.common.utils import configure_logging
+from motleycrew.common import configure_logging
 from motleycrew.tasks import SimpleTask
 
 

examples/_test_single_openai_tools_react.py

@@ -4,7 +4,7 @@
 from motleycrew import MotleyCrew
 from motleycrew.agents.langchain.openai_tools_react import ReactOpenAIToolsAgent
 from motleycrew.agents.langchain.react import ReactMotleyAgent
-from motleycrew.common.utils import configure_logging
+from motleycrew.common import configure_logging
 from motleycrew.tasks import SimpleTask
 from motleycache import enable_cache
 

examples/delegation_crewai.py

@@ -12,7 +12,7 @@
 from motleycrew import MotleyCrew
 from motleycrew.agents.crewai import CrewAIMotleyAgent
 from motleycrew.agents.langchain.react import ReactMotleyAgent
-from motleycrew.common.utils import configure_logging
+from motleycrew.common import configure_logging
 from motleycrew.tasks import SimpleTask
 
 WORKING_DIR = Path(os.path.realpath("."))

examples/delegation_demo.py

@@ -13,7 +13,7 @@
 from motleycrew.agents.langchain.react import ReactMotleyAgent
 from motleycrew.agents.llama_index import ReActLlamaIndexMotleyAgent
 from motleycrew.tools.image.dall_e import DallEImageGeneratorTool
-from motleycrew.common.utils import configure_logging
+from motleycrew.common import configure_logging
 from motleycrew.tasks import SimpleTask
 
 WORKING_DIR = Path(os.path.realpath("."))

examples/image_generation_crewai.py

@@ -3,7 +3,7 @@
 from motleycrew import MotleyCrew
 from motleycrew.agents.crewai import CrewAIMotleyAgent
 from motleycrew.tools.image.dall_e import DallEImageGeneratorTool
-from motleycrew.common.utils import configure_logging
+from motleycrew.common import configure_logging
 
 
 def main():

examples/math_crewai.py

@@ -3,7 +3,7 @@
 from motleycrew import MotleyCrew
 from motleycrew.agents.crewai import CrewAIMotleyAgent
 from motleycrew.tools import PythonREPLTool
-from motleycrew.common.utils import configure_logging
+from motleycrew.common import configure_logging
 
 
 def main():

examples/research_agent/research_agent_main.py

@@ -8,7 +8,7 @@
 
 from motleycrew import MotleyCrew
 from motleycrew.storage import MotleyKuzuGraphStore
-from motleycrew.common.utils import configure_logging
+from motleycrew.common import configure_logging
 from motleycrew.applications.research_agent.question_task import QuestionTask
 from motleycrew.applications.research_agent.answer_task import AnswerTask
 

examples/test_single_crewai_agent.py

@@ -3,7 +3,7 @@
 
 from motleycrew import MotleyCrew
 from motleycrew.agents.crewai import CrewAIMotleyAgent
-from motleycrew.common.utils import configure_logging
+from motleycrew.common import configure_logging
 
 
 def main():

motleycrew/agents/parent.py

@@ -1,4 +1,3 @@
-import logging
 from typing import TYPE_CHECKING, Optional, Sequence
 
 from langchain_core.tools import Tool
@@ -9,6 +8,7 @@
 from motleycrew.tools import MotleyTool
 from motleycrew.common import MotleyAgentFactory, MotleySupportedTool
 from motleycrew.common.exceptions import AgentNotMaterialized, CannotModifyMaterializedAgent
+from motleycrew.common import logger
 
 if TYPE_CHECKING:
     from motleycrew import MotleyCrew
@@ -57,7 +57,7 @@ def is_materialized(self):
 
     def materialize(self):
         if self.is_materialized:
-            logging.info("Agent is already materialized, skipping materialization")
+            logger.info("Agent is already materialized, skipping materialization")
             return
         assert self.agent_factory, "Cannot materialize agent without a factory provided"
         self._agent = self.agent_factory(tools=self.tools)
@@ -91,7 +91,7 @@ def call_agent(*args, **kwargs):
         )
 
     # def call_as_tool(self, *args, **kwargs) -> Any:
-    #     logging.info("Entering delegation for %s", self.name)
+    #     logger.info("Entering delegation for %s", self.name)
     #     assert self.crew, "can't accept delegated task outside of a crew"
     #
     #     if len(args) > 0:
@@ -102,7 +102,7 @@ def call_agent(*args, **kwargs):
     #     else:
     #         input_ = json.dumps(kwargs)
     #
-    #     logging.info("Made the args: %s", input_)
+    #     logger.info("Made the args: %s", input_)
     #
     #     # TODO: pass context of parent task to agent nicely?
     #     # TODO: mark the current task as depending on the new task
@@ -117,11 +117,11 @@ def call_agent(*args, **kwargs):
     #     )
     #
     #     # TODO: make sure tools return task objects, which are properly used by callers
-    #     logging.info("Executing subtask '%s'", task.name)
+    #     logger.info("Executing subtask '%s'", task.name)
     #     self.crew.task_graph.set_task_running(task=task)
     #     result = self.crew.execute(task, return_result=True)
     #
-    #     logging.info("Finished subtask '%s' - %s", task.name, result)
+    #     logger.info("Finished subtask '%s' - %s", task.name, result)
     #     self.crew.task_graph.set_task_done(task=task)
     #
     #     return result

motleycrew/applications/research_agent/answer_task.py

@@ -1,4 +1,3 @@
-import logging
 from typing import List, Optional
 from langchain_core.runnables import Runnable
 
@@ -10,6 +9,7 @@
 from motleycrew.applications.research_agent.question import Question, QuestionAnsweringTaskUnit
 from motleycrew.applications.research_agent.question_answerer import AnswerSubQuestionTool
 from motleycrew.storage import MotleyGraphStore
+from motleycrew.common import logger
 
 
 class AnswerTask(Task):
@@ -34,7 +34,7 @@ def get_next_unit(self) -> QuestionAnsweringTaskUnit | None:
         ).format(Question.get_label(), Question.get_label())
 
         query_result = self.graph_store.run_cypher_query(query, container=Question)
-        logging.info("Available questions: %s", query_result)
+        logger.info("Available questions: %s", query_result)
         if not query_result:
             return None
         else:

motleycrew/applications/research_agent/question_generator.py

@@ -1,6 +1,5 @@
 from typing import Optional
 from pathlib import Path
-import logging
 
 from langchain_core.language_models import BaseLanguageModel
 from langchain_core.runnables import (
@@ -18,6 +17,7 @@
 from motleycrew.common.llms import init_llm
 from motleycrew.common.utils import print_passthrough
 from motleycrew.storage import MotleyGraphStore
+from motleycrew.common import logger
 
 
 from motleycrew.applications.research_agent.question import Question, QuestionGenerationTaskUnit
@@ -103,10 +103,10 @@ def insert_questions(input_dict) -> None:
         questions_raw = input_dict["subquestions"].content
         questions = [q.strip() for q in questions_raw.split("\n") if len(q.strip()) > 1]
         for q in questions:
-            logging.info("Inserting question: %s", q)
+            logger.info("Inserting question: %s", q)
             subquestion = graph.insert_node(Question(question=q))
             graph.create_relation(input_dict["question"], subquestion, IS_SUBQUESTION_PREDICATE)
-        logging.info("Inserted %s questions", len(questions))
+        logger.info("Inserted %s questions", len(questions))
 
     def set_context(input_dict: dict):
         node = input_dict["question"]

motleycrew/applications/research_agent/question_task.py

@@ -1,4 +1,3 @@
-import logging
 from typing import List, Optional
 
 from langchain_core.runnables import Runnable
@@ -10,6 +9,7 @@
 from .question import Question, QuestionGenerationTaskUnit
 from .question_generator import QuestionGeneratorTool
 from .question_prioritizer import QuestionPrioritizerTool
+from motleycrew.common import logger
 
 
 class QuestionTask(Task):
@@ -39,7 +39,7 @@ def get_next_unit(self) -> QuestionGenerationTaskUnit | None:
             return None
 
         unanswered_questions = self.get_unanswered_questions(only_without_children=True)
-        logging.info("Loaded unanswered questions: %s", unanswered_questions)
+        logger.info("Loaded unanswered questions: %s", unanswered_questions)
 
         if not len(unanswered_questions):
             return None
@@ -50,11 +50,11 @@ def get_next_unit(self) -> QuestionGenerationTaskUnit | None:
                 "unanswered_questions": unanswered_questions,
             }
         )
-        logging.info("Most pertinent question according to the tool: %s", most_pertinent_question)
+        logger.info("Most pertinent question according to the tool: %s", most_pertinent_question)
         return QuestionGenerationTaskUnit(question=most_pertinent_question)
 
     def register_completed_unit(self, unit: TaskUnitType) -> None:
-        logging.info("==== Completed iteration %s of %s ====", self.n_iter + 1, self.max_iter)
+        logger.info("==== Completed iteration %s of %s ====", self.n_iter + 1, self.max_iter)
         self.n_iter += 1
         if self.n_iter >= self.max_iter:
             self.set_done(True)

motleycrew/common/__init__.py

@@ -9,3 +9,5 @@
 
 from .types import MotleySupportedTool
 from .types import MotleyAgentFactory
+
+from .logging import logger, configure_logging

motleycrew/common/logging.py

@@ -0,0 +1,18 @@
+import logging
+
+
+# init logger
+logger = logging.getLogger("motleycrew_logger")
+stream_handler = logging.StreamHandler()
+formatter = logging.Formatter(fmt="%(asctime)s - %(name)s - %(levelname)s - %(message)s")
+stream_handler.setFormatter(formatter)
+logger.addHandler(stream_handler)
+
+
+def configure_logging(verbose: bool = False, debug: bool = False):
+    if debug:
+        logger.setLevel(logging.DEBUG)
+    elif verbose:
+        logger.setLevel(logging.INFO)
+    else:
+        logger.setLevel(logging.WARNING)

motleycrew/common/utils.py

@@ -1,23 +1,12 @@
 import sys
 from typing import Optional, Sequence
-import logging
 import hashlib
 from urllib.parse import urlparse
 from langchain_core.messages import BaseMessage
 
 from motleycrew.common.exceptions import ModuleNotInstalledException
 
 
-def configure_logging(verbose: bool = False, debug: bool = False):
-    if debug:
-        level = logging.DEBUG
-    elif verbose:
-        level = logging.INFO
-    else:
-        level = logging.WARNING
-    logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s", level=level)
-
-
 def to_str(value: str | BaseMessage | Sequence[str] | Sequence[BaseMessage]) -> str:
     if isinstance(value, str):
         return value

motleycrew/crew.py

@@ -1,12 +1,12 @@
 from typing import Collection, Sequence, Optional
-import logging
 import os
 
 from motleycrew.agents.parent import MotleyAgentParent
 from motleycrew.tasks import Task, TaskUnit, SimpleTask
 from motleycrew.storage import MotleyGraphStore
 from motleycrew.storage.graph_store_utils import init_graph_store
 from motleycrew.tools import MotleyTool
+from motleycrew.common import logger
 
 
 class MotleyCrew:
@@ -39,7 +39,7 @@ def create_simple_task(
 
     def run(self) -> list[TaskUnit]:
         if not self.single_thread:
-            logging.warning("Multithreading is not implemented yet, will run in single thread")
+            logger.warning("Multithreading is not implemented yet, will run in single thread")
 
         return self._run_sync()
 
@@ -70,32 +70,32 @@ def _run_sync(self) -> list[TaskUnit]:
             did_something = False
 
             available_tasks = self.get_available_tasks()
-            logging.info("Available tasks: %s", available_tasks)
+            logger.info("Available tasks: %s", available_tasks)
 
             for task in available_tasks:
-                logging.info("Processing task: %s", task)
+                logger.info("Processing task: %s", task)
 
                 next_unit = task.get_next_unit()
 
                 if next_unit is None:
-                    logging.info("Got no matching units for task %s", task)
+                    logger.info("Got no matching units for task %s", task)
                 else:
-                    logging.info("Got a matching unit for task %s", task)
+                    logger.info("Got a matching unit for task %s", task)
                     current_unit = next_unit
-                    logging.info("Processing task: %s", current_unit)
+                    logger.info("Processing task: %s", current_unit)
 
                     extra_tools = self.get_extra_tools(task)
 
                     agent = task.get_worker(extra_tools)
-                    logging.info("Assigned unit %s to agent %s, dispatching", current_unit, agent)
+                    logger.info("Assigned unit %s to agent %s, dispatching", current_unit, agent)
                     current_unit.set_running()
                     task.register_started_unit(current_unit)
 
                     # TODO: accept and handle some sort of return value? Or just the final state of the task?
                     result = agent.invoke(current_unit.as_dict())
                     current_unit.output = result
 
-                    logging.info("Task unit %s completed, marking as done", current_unit)
+                    logger.info("Task unit %s completed, marking as done", current_unit)
                     current_unit.set_done()
                     task.register_completed_unit(current_unit)
                     done_units.append(current_unit)
@@ -104,7 +104,7 @@ def _run_sync(self) -> list[TaskUnit]:
                     continue
 
             if not did_something:
-                logging.info("Nothing left to do, exiting")
+                logger.info("Nothing left to do, exiting")
                 return done_units
 
     def get_available_tasks(self) -> list[Task]:
@@ -138,7 +138,7 @@ def get_available_tasks(self) -> list[Task]:
     #                 raise exc
     #
     #             task = future.mc_task
-    #             logging.info(f"Finished task '{task.name}'")
+    #             logger.info(f"Finished task '{task.name}'")
     #             self.futures.remove(future)
     #             tasks.set_task_done(task)
     #             self.adispatch_next_batch()
@@ -150,7 +150,7 @@ def get_available_tasks(self) -> list[Task]:
     #     next_ = self.task_graph.get_ready_tasks()
     #     for t in next_:
     #         self.task_graph.set_task_running(t)
-    #         logging.info(f"Dispatching task '{t.name}'")
+    #         logger.info(f"Dispatching task '{t.name}'")
     #         future = self.thread_pool.submit(
     #             self.execute,
     #             t,

motleycrew/storage/graph_store_utils.py

@@ -1,11 +1,10 @@
 import tempfile
 from typing import Optional
-import logging
 import os
 
 from motleycrew.common import Defaults
 from motleycrew.common import GraphStoreType
-
+from motleycrew.common import logger
 from motleycrew.storage import MotleyKuzuGraphStore
 
 
@@ -17,10 +16,10 @@ def init_graph_store(
         import kuzu
 
         if db_path is None:
-            logging.info("No db_path provided, creating temporary directory for database")
+            logger.info("No db_path provided, creating temporary directory for database")
             db_path = os.path.join(tempfile.mkdtemp(), "kuzu_db")
 
-        logging.info("Using Kuzu graph store with path: %s", db_path)
+        logger.info("Using Kuzu graph store with path: %s", db_path)
         db = kuzu.Database(db_path)
         return MotleyKuzuGraphStore(db)