Merge pull request #784 from TransformerOptimus/twitter_tool_fix

Twitter tool fix

superagi/agent/output_parser.py

@@ -23,11 +23,14 @@ class BaseOutputParser(ABC):
     def parse(self, text: str) -> AgentGPTAction:
         """Return AgentGPTAction"""
 
+
 class AgentSchemaOutputParser(BaseOutputParser):
     def parse(self, response: str) -> AgentGPTAction:
         if response.startswith("```") and response.endswith("```"):
             response = "```".join(response.split("```")[1:-1])
-            response = JsonCleaner.extract_json_section(response)
+        response = JsonCleaner.extract_json_section(response)
+        # ast throws error if true/false params passed in json
+        response = JsonCleaner.clean_boolean(response)
 
         # OpenAI returns `str(content_dict)`, literal_eval reverses this
         try:
@@ -40,76 +43,3 @@ def parse(self, response: str) -> AgentGPTAction:
         except BaseException as e:
             logger.info(f"AgentSchemaOutputParser: Error parsing JSON respons {e}")
             return {}
-
-class AgentOutputParser(BaseOutputParser):
-    def parse(self, text: str) -> AgentGPTAction:
-        try:
-            logger.info(text)
-            text = JsonCleaner.check_and_clean_json(text)
-            parsed = json5.loads(text)
-        except json.JSONDecodeError:
-            return AgentGPTAction(
-                name="ERROR",
-                args={"error": f"Could not parse invalid json: {text}"},
-            )
-        try:
-            format_prefix_yellow = "\033[93m\033[1m"
-            format_suffix_yellow = "\033[0m\033[0m"
-            format_prefix_green = "\033[92m\033[1m"
-            format_suffix_green = "\033[0m\033[0m"
-            logger.info(format_prefix_green + "Intelligence : " + format_suffix_green)
-            if "text" in parsed["thoughts"]:
-                logger.info(format_prefix_yellow + "Thoughts: " + format_suffix_yellow + parsed["thoughts"]["text"] + "\n")
-
-            if "reasoning" in parsed["thoughts"]:
-                logger.info(format_prefix_yellow + "Reasoning: " + format_suffix_yellow + parsed["thoughts"]["reasoning"] + "\n")
-
-            if "plan" in parsed["thoughts"]:
-                logger.info(format_prefix_yellow + "Plan: " + format_suffix_yellow + str(parsed["thoughts"]["plan"]) + "\n")
-
-            if "criticism" in parsed["thoughts"]:
-                logger.info(format_prefix_yellow + "Criticism: " + format_suffix_yellow + parsed["thoughts"]["criticism"] + "\n")
-
-            logger.info(format_prefix_green + "Action : " + format_suffix_green)
-            # print(format_prefix_yellow + "Args: "+ format_suffix_yellow + parsed["tool"]["args"] + "\n")
-            if "tool" not in parsed:
-                raise Exception("No tool found in the response..")
-            if parsed["tool"] is None or not parsed["tool"]:
-                return AgentGPTAction(name="", args="")
-            if "name" in parsed["tool"]:
-                logger.info(format_prefix_yellow + "Tool: " + format_suffix_yellow + parsed["tool"]["name"] + "\n")
-            args = {}
-            if "args" in parsed["tool"]:
-                args = parsed["tool"]["args"]
-            return AgentGPTAction(
-                name=parsed["tool"]["name"],
-                args=args,
-            )
-        except (KeyError, TypeError) as e:
-            logger.error(f"Error parsing output:", e)
-            # If the tool is null or incomplete, return an erroneous tool
-            return AgentGPTAction(
-                name="ERROR", args={"error": f"Unable to parse the output: {parsed}"}
-            )
-
-    def parse_tasks(self, text: str) -> AgentTasks:
-        try:
-            parsed = json.loads(text, strict=False)
-        except json.JSONDecodeError:
-            preprocessed_text = JsonCleaner.preprocess_json_input(text)
-            try:
-                parsed = json.loads(preprocessed_text, strict=False)
-            except Exception:
-                return AgentTasks(
-                    error=f"Could not parse invalid json: {text}",
-                )
-        try:
-            logger.info("Tasks: ", parsed["tasks"])
-            return AgentTasks(
-                tasks=parsed["tasks"]
-            )
-        except (KeyError, TypeError):
-            # If the command is null or incomplete, return an erroneous tool
-            return AgentTasks(
-                error=f"Incomplete tool args: {parsed}",
-            )

superagi/agent/super_agi.py

@@ -6,15 +6,15 @@
 import time
 from typing import Any
 from typing import Tuple
-import json
+
 import numpy as np
 from pydantic import ValidationError
 from pydantic.types import List
 from sqlalchemy import asc
 from sqlalchemy.orm import sessionmaker
 
 from superagi.agent.agent_prompt_builder import AgentPromptBuilder
-from superagi.agent.output_parser import BaseOutputParser, AgentOutputParser, AgentSchemaOutputParser
+from superagi.agent.output_parser import BaseOutputParser, AgentSchemaOutputParser
 from superagi.agent.task_queue import TaskQueue
 from superagi.apm.event_handler import EventHandler
 from superagi.helper.token_counter import TokenCounter
@@ -65,23 +65,6 @@ def __init__(self,
         self.agent_config = agent_config
         self.agent_execution_config = agent_execution_config
 
-    @classmethod
-    def from_llm_and_tools(
-            cls,
-            ai_name: str,
-            ai_role: str,
-            memory: VectorStore,
-            tools: List[BaseTool],
-            llm: BaseLlm
-    ) -> SuperAgi:
-        return cls(
-            ai_name=ai_name,
-            ai_role=ai_role,
-            llm=llm,
-            memory=memory,
-            output_parser=AgentOutputParser(),
-            tools=tools
-        )
 
     def fetch_agent_feeds(self, session, agent_execution_id):
         agent_feeds = session.query(AgentExecutionFeed.role, AgentExecutionFeed.feed) \

superagi/helper/json_cleaner.py

@@ -8,60 +8,20 @@
 class JsonCleaner:
 
     @classmethod
-    def check_and_clean_json(cls, json_string: str):
+    def clean_boolean(cls, input_str: str = ""):
         """
-        Checks if the given string is a valid json string.
-        If not, tries to clean it up and return a valid json string.
+        Clean the boolean values in the given string.
 
         Args:
-            json_string (str): The json string to be checked and cleaned.
+            input_str (str): The string from which the json section is to be extracted.
 
         Returns:
-            str: The cleaned json string.
-        """
-        try:
-            json_string = cls.remove_escape_sequences(json_string)
-            json_string = cls.remove_trailing_newline_spaces(json_string)
-            json_string = cls.clean_newline_characters(json_string)
-
-            json5.loads(json_string)
-            return json_string
-        except (ValueError, json.JSONDecodeError) as e:
-            # If the json is invalid, try to clean it up
-            json_string = cls.extract_json_section(json_string)
-            json_string = cls.preprocess_json_input(json_string)
-            json_string = cls.add_quotes_to_property_names(json_string)
-            json_string = cls.remove_escape_sequences(json_string)
-            # call this post remove_escape_sequences
-            json_string = cls.clean_newline_characters(json_string)
-            json_string = cls.balance_braces(json_string)
-            try:
-                json5.loads(json_string)
-                return json_string
-            except (ValueError, json.JSONDecodeError) as e:
-                logger.info(json_string)
-                # If the json is still invalid, try to extract the json section
-                json_string = cls.extract_json_section(json_string)
-                return json_string
-        return json_string
-
-    @classmethod
-    def preprocess_json_input(cls, input_str: str) -> str:
+            str: The extracted json section.
         """
-        Preprocess the given json string.
-        Replace single backslashes with double backslashes,
-        while leaving already escaped ones intact.
-
-        Args:
-            input_str (str): The json string to be preprocessed.
+        input_str = re.sub(r':\s*false', ': False', input_str)
+        input_str = re.sub(r':\s*true', ': True', input_str)
+        return input_str
 
-        Returns:
-            str: The preprocessed json string.
-        """
-        corrected_str = re.sub(
-            r'(?<!\\)\\(?!["\\/bfnrt]|u[0-9a-fA-F]{4})', r"\\\\", input_str
-        )
-        return corrected_str
 
     @classmethod
     def extract_json_section(cls, input_str: str = ""):
@@ -97,40 +57,6 @@ def remove_escape_sequences(cls, string):
         """
         return string.encode('utf-8').decode('unicode_escape').encode('raw_unicode_escape').decode('utf-8')
 
-    @classmethod
-    def remove_trailing_newline_spaces(cls, json_string):
-        json_string = re.sub(r'\n+\s*"', '"', json_string)
-        json_string = re.sub(r'}\n+\s*}', '}}', json_string)
-        json_string = re.sub(r'"\n+\s*}', '"}', json_string)
-        json_string = re.sub(r'\n+\s*}', '}', json_string)
-        json_string = re.sub(r'\n+\s*]', ']', json_string)
-        return json_string.strip()
-
-    @classmethod
-    def clean_newline_characters(cls, string):
-        string = string.replace("\t", "\\t")
-        string = string.replace("\n", "\\n")
-        return string
-
-    @classmethod
-    def add_quotes_to_property_names(cls, json_string: str) -> str:
-        """
-        Add quotes to property names in the given json string.
-
-        Args:
-            json_string (str): The json string to be processed.
-
-        Returns:
-            str: The json string with quotes added to property names.
-        """
-
-        def replace(match: re.Match) -> str:
-            return f'{match.group(1)}"{match.group(2)}":'
-
-        json_string = re.sub(r'([,{])\n*\s*(\b\w+\b):', replace, json_string)
-
-        return json_string
-
     @classmethod
     def balance_braces(cls, json_string: str) -> str:
         """

superagi/lib/logger.py

@@ -52,27 +52,27 @@ def _make_custom_log_record(self, name, level, fn, lno, msg, args, exc_info, fun
 
     def debug(self, message, *args):
         self.logger.debug(message)
-        if len(args) > 0:
+        if args:
             self.logger.debug(*args)
 
     def info(self, message, *args):
         self.logger.info(message)
-        if len(args) > 0:
+        if args:
             self.logger.info(*args)
 
     def warning(self, message, *args):
         self.logger.warning(message)
-        if len(args) > 0:
+        if args:
             self.logger.warning(*args)
 
     def error(self, message, *args):
         self.logger.error(message)
-        if len(args) > 0:
+        if args:
             self.logger.error(*args)
 
     def critical(self, message, *args):
         self.logger.critical(message)
-        if len(args) > 0:
+        if args:
             self.logger.critical(*args)
 
 

superagi/tools/code/write_code.py

@@ -99,7 +99,7 @@ def _execute(self, code_description: str) -> str:
 
         # Get README contents and save
         split_result = result["content"].split("```")
-        if len(split_result) > 0:
+        if split_result:
             readme = split_result[0]
             save_readme_result = self.resource_manager.write_file("README.md", readme)
             if save_readme_result.startswith("Error"):

superagi/tools/google_serp_search/google_serp_search.py

@@ -51,7 +51,7 @@ def _execute(self, query: str) -> tuple:
         serp_api = GoogleSerpApiWrap(api_key)
         response = serp_api.search_run(query)
         summary = self.summarise_result(query, response["snippets"])
-        if len(response["links"]) > 0:
+        if response["links"]:
             return summary + "\n\nLinks:\n" + "\n".join("- " + link for link in response["links"][:3])
         return summary
 

superagi/tools/jira/edit_issue.py

@@ -44,7 +44,7 @@ def _execute(self, key: str, fields: dict):
         """
         jira = self.build_jira_instance()
         issues = jira.search_issues('key=')
-        if len(issues) > 0:
+        if issues:
             issues[0].update(fields=fields)
             return f"Issue '{issues[0].key}' created successfully!"
         return f"Issue not found!"

tests/unit_tests/agent/test_output_parser.py

@@ -1,9 +1,10 @@
 import pytest
-from superagi.agent.output_parser import AgentOutputParser, AgentGPTAction, AgentTasks
+
+from superagi.agent.output_parser import AgentGPTAction, AgentSchemaOutputParser
 
 
 def test_parse():
-    parser = AgentOutputParser()
+    parser = AgentSchemaOutputParser()
 
     # test with valid input
     valid_text = '{"thoughts": {"text": "some thought", "reasoning": "some reasoning", "plan": "some plan", "criticism": "some criticism"}, "tool": {"name": "some tool", "args": {"arg1": "value1"}}}'
@@ -12,31 +13,3 @@ def test_parse():
     assert output.name == "some tool"
     assert output.args == {"arg1": "value1"}
 
-    # test with invalid json
-    invalid_json = '{"this is not valid json'
-    with pytest.raises(ValueError):
-        parser.parse(invalid_json)
-
-
-def test_parse_tasks():
-    parser = AgentOutputParser()
-
-    # test with valid input
-    valid_text = '{"tasks": [{"task1": "value1"}, {"task2": "value2"}]}'
-    output = parser.parse_tasks(valid_text)
-    assert isinstance(output, AgentTasks)
-    assert output.tasks == [{"task1": "value1"}, {"task2": "value2"}]
-    assert output.error == ""
-
-    # test with incomplete tool args
-    invalid_tasks = '{"tasks": []}'
-    output = parser.parse_tasks(invalid_tasks)
-    assert isinstance(output, AgentTasks)
-    assert output.tasks == []
-    assert output.error == ""
-
-    # test with invalid json
-    invalid_json = '{"this is not valid json'
-    output = parser.parse_tasks(invalid_json)
-    assert isinstance(output, AgentTasks)
-    assert "Could not parse invalid json" in output.error

tests/unit_tests/agent_permissions/test_check_permission_in_restricted_mode.py

@@ -1,6 +1,8 @@
-import pytest
 from unittest.mock import MagicMock, Mock
-from superagi.agent.output_parser import AgentOutputParser
+
+import pytest
+
+from superagi.agent.output_parser import AgentSchemaOutputParser
 from superagi.agent.super_agi import SuperAgi
 from superagi.llms.base_llm import BaseLlm
 from superagi.tools.base_tool import BaseTool
@@ -31,7 +33,7 @@ def super_agi():
     tools = [MockTool(name="NotRestrictedTool", permission_required=False),
              MockTool(name="RestrictedTool", permission_required=True)]
     agent_config = {"permission_type": "RESTRICTED", "agent_execution_id": 1, "agent_id": 2}
-    output_parser = AgentOutputParser()
+    output_parser = AgentSchemaOutputParser()
 
     super_agi = SuperAgi(ai_name, ai_role, llm, memory, tools, agent_config, output_parser)
     return super_agi