tool_sandbox/notebooks/playground_single_run_analysis.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Notebook for analyzing a tool sandbox run"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pathlib\n",
    "\n",
    "import pandas as pd\n",
    "import polars as pl\n",
    "import seaborn as sns\n",
    "\n",
    "from tool_sandbox.analysis.analysis import extract_meta_stats, extract_aggregated_stats\n",
    "from tool_sandbox.analysis.data_loading import (\n",
    "    extract_scenario_results,\n",
    "    get_scenario_pretty_print_path,\n",
    "    load_result_summary,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "result_summary_path = pathlib.Path(\n",
    "    # \"/Users/tholleis/code/ToolSandbox/data/agent_gemini-1.5-pro-001_user_gpt-4o-2024-05-13_06_14_2024_19_56_00_no_tool_assert/result_summary.json\"\n",
    "    \"/Users/tholleis/code/ToolSandbox/data/agent_gemini-1.5-pro-001_user_gpt-4o-2024-05-13_06_14_2024_20_20_49/result_summary.json\"\n",
    ")\n",
    "\n",
    "results = load_result_summary(result_summary_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "scenario_df = extract_scenario_results(results)\n",
    "scenario_df.head(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Extract high-level metadata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "meta_stats = extract_meta_stats(scenario_df)\n",
    "meta_stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "exceptions_df = scenario_df.filter(pl.col(\"exception_type\").is_not_null())\n",
    "exception_count_df = (\n",
    "    exceptions_df.get_column(\"exception_type\")\n",
    "    .value_counts()\n",
    "    .sort(by=\"count\", descending=True)\n",
    ")\n",
    "exception_count_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Analyze the different types of exceptions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_link_to_pretty_print_file(scenario_name: str):\n",
    "    pretty_print_path = get_scenario_pretty_print_path(\n",
    "        result_summary_path=result_summary_path, scenario_name=scenario_name\n",
    "    )\n",
    "    return f'<a href=\"{pretty_print_path}\">{scenario_name}</a>'\n",
    "\n",
    "\n",
    "for exception_type in exception_count_df.get_column(\"exception_type\").to_list():\n",
    "    df = exceptions_df.filter(pl.col(\"exception_type\") == exception_type)\n",
    "    pd_df = df.to_pandas()\n",
    "    desired_column_order = [\n",
    "        \"exception_type\",\n",
    "        \"name\",\n",
    "        \"traceback\",\n",
    "        \"milestone_similarity\",\n",
    "        \"minefield_similarity\",\n",
    "        \"similarity\",\n",
    "        \"turn_count\",\n",
    "        \"categories\",\n",
    "        \"milestone_mapping\",\n",
    "        \"minefield_mapping\",\n",
    "    ]\n",
    "    pd_df = pd_df[desired_column_order]\n",
    "    display(\n",
    "        pd_df.style.format({\"name\": create_link_to_pretty_print_file})\n",
    "        .set_caption(f\"{exception_type} | {len(df)} / {len(exceptions_df)} exceptions\")\n",
    "        .set_properties(**{\"text-align\": \"left\"})\n",
    "    )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Extract aggregated statistics per scenario category"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "agg_stats_df = extract_aggregated_stats(results)\n",
    "with pl.Config(tbl_rows=-1):\n",
    "    display(agg_stats_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Analyze the top/bottom N samples for each category."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "scenario_exploded_categories_df = scenario_df.explode(\"categories\")\n",
    "scenario_exploded_categories_df = scenario_exploded_categories_df.rename(\n",
    "    {\"categories\": \"category\"}\n",
    ")\n",
    "scenario_exploded_categories_df.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Let's ignore the augmentation categories for now.\n",
    "CATEGORIES_WITH_AUGMENTATION = {\n",
    "    ScenarioCategories.NO_DISTRACTION_TOOLS,\n",
    "    ScenarioCategories.THREE_DISTRACTION_TOOLS,\n",
    "    ScenarioCategories.TEN_DISTRACTION_TOOLS,\n",
    "    ScenarioCategories.ALL_TOOLS_AVAILABLE,\n",
    "    ScenarioCategories.TOOL_NAME_SCRAMBLED,\n",
    "    ScenarioCategories.TOOL_DESCRIPTION_SCRAMBLED,\n",
    "    ScenarioCategories.ARG_DESCRIPTION_SCRAMBLED,\n",
    "    ScenarioCategories.ARG_NAME_SCRAMBLED,\n",
    "    ScenarioCategories.ARG_TYPE_SCRAMBLED,\n",
    "}\n",
    "CATEGORIES_WITHOUT_AUGMENTATION = {\n",
    "    category\n",
    "    for category in ScenarioCategories\n",
    "    if category not in CATEGORIES_WITH_AUGMENTATION\n",
    "}\n",
    "\n",
    "category_strs = {str(c) for c in CATEGORIES_WITHOUT_AUGMENTATION}\n",
    "\n",
    "# We want to look at scenarios with high/low similarity scores, but not ones where an\n",
    "# exception occurred.\n",
    "scenario_exploded_categories_pd_df = scenario_exploded_categories_df.filter(\n",
    "    pl.col(\"exception_type\").is_null()\n",
    ").to_pandas()\n",
    "pd_df = scenario_exploded_categories_pd_df[\n",
    "    scenario_exploded_categories_pd_df[\"category\"].isin(category_strs)\n",
    "]\n",
    "\n",
    "# Show the worst N samples with the lowest similarity for each scenario category.\n",
    "pd_df = pd_df.sort_values([\"category\", \"similarity\"], ascending=[True, True])\n",
    "N = 3\n",
    "bottom_n_per_category = pd_df.groupby(\"category\").head(N)\n",
    "with pd.option_context(\"display.max_colwidth\", None):\n",
    "    display(\n",
    "        bottom_n_per_category.style.format({\"name\": create_link_to_pretty_print_file})\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Show the best N samples with the highest similarity for each scenario category.\n",
    "pd_df = pd_df.sort_values([\"category\", \"similarity\"], ascending=[True, False])\n",
    "N = 1\n",
    "top_n_per_category = pd_df.groupby(\"category\").head(N)\n",
    "with pd.option_context(\"display.max_colwidth\", None):\n",
    "    # From https://stackoverflow.com/a/51855581 : we want to make the scenario name a\n",
    "    # clickable link to the `pretty_print.txt` file.\n",
    "    display(top_n_per_category.style.format({\"name\": create_link_to_pretty_print_file}))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "g = sns.catplot(\n",
    "    data=scenario_exploded_categories_pd_df,\n",
    "    x=\"similarity\",\n",
    "    y=\"category\",\n",
    "    kind=\"strip\",\n",
    "    height=10,\n",
    ")\n",
    "g.set_axis_labels(\"Similarity\", \"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "ToolSandbox",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.19"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}