exps 1-3 with llama 3.1 using fireworks api

mdagent/agent/agent.py

@@ -78,6 +78,7 @@ def _initialize_tools_and_agent(self, user_input=None):
                 self.tools = get_tools(
                     query=user_input,
                     llm=self.tools_llm,
+                    top_k_tools=self.top_k_tools,
                     human=self.use_human_tool,
                 )
             else:
@@ -103,6 +104,7 @@ def run(self, user_input, callbacks=None):
         elif self.agent_type == "OpenAIFunctionsAgent":
             self.prompt = openaifxn_prompt.format(input=user_input, context=run_memory)
         self.agent = self._initialize_tools_and_agent(user_input)
+        print("Num of tokens:", self.llm.get_num_tokens(self.prompt))
         model_output = self.agent.invoke(self.prompt, callbacks=callbacks)
         if self.use_memory:
             self.memory.generate_agent_summary(model_output)

mdagent/agent/memory.py

@@ -128,7 +128,6 @@ def generate_agent_summary(self, agent_trace):
         Returns:
         - None
         """
-        print(agent_trace)
         llm_out = self.llm_agent_trace.invoke({"agent_trace": agent_trace})
         key_str = f"{self.run_id}.{self.get_summary_number()}"
         run_summary = {key_str: llm_out}

mdagent/utils/makellm.py

@@ -1,6 +1,5 @@
 from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler
 from langchain_openai import ChatOpenAI
-from langchain_together import ChatTogether
 
 
 def _make_llm(model, temp, verbose):
@@ -12,14 +11,25 @@ def _make_llm(model, temp, verbose):
             streaming=True if verbose else False,
             callbacks=[StreamingStdOutCallbackHandler()] if verbose else None,
         )
-    elif model.startswith("Meta-Llama"):
-        llm = ChatTogether(
+    elif model.startswith("llama"):
+        from langchain_fireworks import ChatFireworks
+
+        llm = ChatFireworks(
             temperature=temp,
-            model=f"meta-llama/{model}",
+            model_name=f"accounts/fireworks/models/{model}",
             request_timeout=1000,
             streaming=True if verbose else False,
             callbacks=[StreamingStdOutCallbackHandler()] if verbose else None,
         )
+    # elif model.startswith("Meta-Llama"):
+    # from langchain_together import ChatTogether
+    # llm = ChatTogether(
+    #     temperature=temp,
+    #     model=f"meta-llama/{model}",
+    #     request_timeout=1000,
+    #     streaming=True if verbose else False,
+    #     callbacks=[StreamingStdOutCallbackHandler()] if verbose else None,
+    # )
     else:
         raise ValueError(f"Invalid or Unsupported model name: {model}")
     return llm

notebooks/experiments/experiment_k1/llama-v3p1-405b-instruct/exp_1.ipynb

@@ -0,0 +1,328 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import datetime\n",
+    "import os\n",
+    "from mdagent import MDAgent"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "date and time: 2024-08-05\n",
+      "time: 16:26:57\n"
+     ]
+    }
+   ],
+   "source": [
+    "#todays date and time\n",
+    "now = datetime.datetime.now()\n",
+    "date = now.strftime(\"%Y-%m-%d\")\n",
+    "print(\"date and time:\",date)\n",
+    "time = now.strftime(\"%H:%M:%S\")\n",
+    "print(\"time:\",time)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "prompt1 = \"Simulate pdb  1MBN at two different temperatures: 300K, 400K for 1ns seconds each. Plot RMSD of both over time, and compare the final secondary structures at the end of the simulations.\"\n",
+    "llm_var = \"llama-v3p1-405b-instruct\"\n",
+    "tools = \"all\"\n",
+    "agent = MDAgent(agent_type=\"Structured\", model=llm_var, top_k_tools=tools)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "LLM:  accounts/fireworks/models/llama-v3p1-405b-instruct \n",
+      "Temperature:  0.1\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(\"LLM: \",agent.llm.model_name,\"\\nTemperature: \",agent.llm.temperature)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Num of tokens: 384\n",
+      "\n",
+      "\n",
+      "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
+      "Thought: First, I need to download the PDB file for 1MBN.\n",
+      "Action: PDBFileDownloader\n",
+      "Action Input: {\"query\": \"1MBN\"}<|eom_id|><|start_header_id|>assistant<|end_header_id|>\n",
+      "\n",
+      "\u001b[32;1m\u001b[1;3mThought: First, I need to download the PDB file for 1MBN.\n",
+      "Action: PDBFileDownloader\n",
+      "Action Input: {\"query\": \"1MBN\"}<|eom_id|><|start_header_id|>assistant<|end_header_id|>\n",
+      "\n",
+      "\u001b[0m\n",
+      "\n",
+      "\u001b[1m> Finished chain.\u001b[0m\n",
+      "Here is a summary of the agent's attempts to solve the problem:\n",
+      "\n",
+      "**Prompt:** Simulate pdb 1MBN at two different temperatures: 300K, 400K for 1ns seconds each. Plot RMSD of both over time, and compare the final secondary structures at the end of the simulations.\n",
+      "\n",
+      "**Step 1:**\n",
+      "Thought: First, I need to download the PDB file for 1MBN.\n",
+      "Action: PDBFileDownloader\n",
+      "Action Input: {\"query\": \"1MBN\"}\n",
+      "\n",
+      "The agent has not yet completed the task, so there are no additional steps or a final solution to report. The agent's next step will likely involve using the downloaded PDB file to set up and run the molecular dynamics simulations at the two specified temperatures.Your run id is:  XCYLSITB\n"
+     ]
+    }
+   ],
+   "source": [
+    "result = agent.run(prompt1)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'Thought: First, I need to download the PDB file for 1MBN.\\nAction: PDBFileDownloader\\nAction Input: {\"query\": \"1MBN\"}<|eom_id|><|start_header_id|>assistant<|end_header_id|>\\n\\n'"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "output, run_id = result\n",
+    "output['output']"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "384\n",
+      "66\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(agent.llm.get_num_tokens(output['input']))\n",
+    "print(agent.llm.get_num_tokens(output['output']))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'0.2.12'"
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import langchain\n",
+    "langchain.__version__"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'error': {'message': 'Input validation error: `inputs` tokens + `max_new_tokens` must be <= 4097. Given: 7338 `inputs` tokens and 4096 `max_new_tokens`',\n",
+       "  'type': 'invalid_request_error',\n",
+       "  'param': 'max_tokens',\n",
+       "  'code': None}}"
+      ]
+     },
+     "execution_count": 9,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "{'error': \n",
+    "    {\n",
+    "        'message': 'Input validation error: `inputs` tokens + `max_new_tokens` must be <= 4097. Given: 7338 `inputs` tokens and 4096 `max_new_tokens`', \n",
+    "        'type': 'invalid_request_error', \n",
+    "        'param': 'max_tokens', \n",
+    "        'code': None\n",
+    "    }\n",
+    "}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "date and time: 2024-08-05\n",
+      "time: 16:27:07\n"
+     ]
+    }
+   ],
+   "source": [
+    "#print final date and time\n",
+    "now = datetime.datetime.now()\n",
+    "date = now.strftime(\"%Y-%m-%d\")\n",
+    "print(\"date and time:\",date)\n",
+    "time = now.strftime(\"%H:%M:%S\")\n",
+    "print(\"time:\",time)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "No names found. The JSON file is empty or does not contain name mappings.\n"
+     ]
+    }
+   ],
+   "source": [
+    "registry = agent.path_registry\n",
+    "paths_and_descriptions = registry.list_path_names_and_descriptions()\n",
+    "print(\"\\n\".join(paths_and_descriptions.split(\",\")))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "ValueError",
+     "evalue": "Cannot embed the '' image format",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
+      "Cell \u001b[0;32mIn[12], line 9\u001b[0m\n\u001b[1;32m      5\u001b[0m path1 \u001b[38;5;241m=\u001b[39m registry\u001b[38;5;241m.\u001b[39mget_mapped_path(rmsd1ID)\n\u001b[1;32m      6\u001b[0m path2 \u001b[38;5;241m=\u001b[39m registry\u001b[38;5;241m.\u001b[39mget_mapped_path(rmsd2ID)\n\u001b[0;32m----> 9\u001b[0m Image(filename\u001b[38;5;241m=\u001b[39mpath1)\n",
+      "File \u001b[0;32m/opt/anaconda3/envs/llama-mdagent/lib/python3.11/site-packages/IPython/core/display.py:961\u001b[0m, in \u001b[0;36mImage.__init__\u001b[0;34m(self, data, url, filename, format, embed, width, height, retina, unconfined, metadata, alt)\u001b[0m\n\u001b[1;32m    958\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39membed \u001b[38;5;241m=\u001b[39m embed \u001b[38;5;28;01mif\u001b[39;00m embed \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;28;01melse\u001b[39;00m (url \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m)\n\u001b[1;32m    960\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39membed \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mformat \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_ACCEPTABLE_EMBEDDINGS:\n\u001b[0;32m--> 961\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mCannot embed the \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m image format\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m%\u001b[39m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mformat))\n\u001b[1;32m    962\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39membed:\n\u001b[1;32m    963\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_mimetype \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_MIMETYPES\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mformat)\n",
+      "\u001b[0;31mValueError\u001b[0m: Cannot embed the '' image format"
+     ]
+    }
+   ],
+   "source": [
+    "#plotting rmsd of both simulations\n",
+    "from IPython.display import Image\n",
+    "rmsd1ID = 'fig0_192900'\n",
+    "rmsd2ID = 'fig0_192903'\n",
+    "path1 = registry.get_mapped_path(rmsd1ID)\n",
+    "path2 = registry.get_mapped_path(rmsd2ID)\n",
+    "\n",
+    "\n",
+    "Image(filename=path1)\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "Image(filename=path2)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Experiment Result:\n",
+    "### Completed without Exception or TimeOut Errors  ❌\n",
+    "### Attempted all necessary steps ✅\n",
+    "### Logic make sense  ✅\n",
+    "### Correct Answer   ❌\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Agent did not realize that it finished"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "mdagent2",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.8"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}