Sibyl-like WebAgent Tool

openai_server/agent_prompting.py

@@ -167,14 +167,8 @@ def agent_system_prompt(agent_code_writer_system_message, agent_system_site_pack
 </data_science>
 Web scraping or web search best practices:
 <web_search>
-* For web search, prioritize using agent_tools provided
-* Do not just use the search snippets to answer questions.  Search snippets are only starting point for finding relevant URLs, documents, or online content.
-* Multi-hop web search is expected, i.e. iterative web search over many turns of a conversation is expected
-* For web search, use ask_question_about_documents.py on promising URLs to answer questions and find new relevant URLs and new relevant documents
-* For web search, use results ask_question_about_documents.py to find new search terms
-* For web search, iterate as many times as required on URLs and documents using web search, ask_question_about_documents.py, and other agent tools
-* For web search multi-hop search, only stop when reaching am answer with information verified and key claims traced to authoritative sources
-* For web search, try to verify your answer with alternative sources to get a reliable answer, especially when user expects a constrained output
+* For web search, prioritize using agent_tools provided.
+* Always prioritize using the web_agent_tool for web-related tasks because it's the most comprehensive web tool.
 </web_search>
 <inline_images>
 Inline image files in response:
@@ -779,6 +773,27 @@ def get_bing_search_helper():
         bing_search = ""
     return bing_search
 
+def get_web_search_helper(model):
+    cwd = os.path.abspath(os.getcwd())
+    have_internet = get_have_internet()
+    if have_internet:
+        os.environ['WEB_TOOL_MODEL'] = model
+        web_search = f"""\n* Search web with web_agent_tool. 
+* For a web related task, you are recommended to use the existing pre-built python code, E.g.:
+```sh
+# filename: my_bing_search.sh
+# execution: true
+python {cwd}/openai_server/agent_tools/web_agent_tool.py --task "WEB_TASK_FOR_THE_AGENT"
+```
+* usage: {cwd}/openai_server/agent_tools/web_agent_tool.py [-h] --task "WEB_TASK_FOR_THE_AGENT"
+* You have to provide a well defined web task for the agent to perform. E.g. instead of "Weather in New York", you should define task as "Get the current weather in New York".
+* This web_agent_tool is a general web agent tool that can be used for any web related task including web search, web scraping, finding information, etc.
+* This web_agent_tool is capable of doing complex search queries from multiple sources and combining the results.
+* This web_agent_tool is the most comprehensive search agent tool available to you, so you always start with this tool when web-related tasks are involved.
+"""
+    else:
+        web_search = ""
+    return web_search
 
 def get_api_helper():
     if os.getenv('SERPAPI_API_KEY') or os.getenv('BING_API_KEY'):
@@ -836,6 +851,7 @@ def get_full_system_prompt(agent_code_writer_system_message, agent_system_site_p
     wolfram_alpha_helper = get_wolfram_alpha_helper()
     news_helper = get_news_api_helper()
     bing_search_helper = get_bing_search_helper()
+    web_search_helper = get_web_search_helper(model)
 
     # general API notes:
     api_helper = get_api_helper()
@@ -869,11 +885,12 @@ def get_full_system_prompt(agent_code_writer_system_message, agent_system_site_p
                             youtube_helper,
                             convert_helper,
                             # search
-                            serp_helper,
+                            web_search_helper,
+                            # serp_helper,
                             semantic_scholar_helper,
                             wolfram_alpha_helper,
                             news_helper,
-                            bing_search_helper,
+                            # bing_search_helper,
                             query_to_web_image_helper,
                             # overall
                             api_helper,