evaluating_generalization_of_truth_directions.ipynb

{
 "cells": [
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-02-27T05:14:15.109893Z",
     "start_time": "2025-02-27T05:14:09.950859Z"
    }
   },
   "cell_type": "code",
   "source": [
    "import json\n",
    "import pandas as pd\n",
    "\n",
    "from tiu_phi_3_5_mini.data_management import load_train_validation_splits, ActivationsDataSelector, ProbeTrainScenario, \\\n",
    "    train_scenarios_spec_path\n",
    "from tiu_phi_3_5_mini.evaluation_utils import MetricsForDatasetProbes, evaluate_classifier_performance\n",
    "from tiu_phi_3_5_mini.logging_setup import create_logger\n",
    "from tiu_phi_3_5_mini.phi_3_5_constants import dsets_index_path, test_classification_metrics_path, analysis_results_folder\n",
    "from tiu_phi_3_5_mini.phi_3_5_probe import ProbesForScenario, load_probes_for_dset"
   ],
   "id": "48018f715646e72a",
   "outputs": [],
   "execution_count": 1
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-02-27T05:14:15.119894Z",
     "start_time": "2025-02-27T05:14:15.115901Z"
    }
   },
   "cell_type": "code",
   "source": "logger = create_logger(__name__)",
   "id": "4cd22df29ccbddcf",
   "outputs": [],
   "execution_count": 2
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-02-27T05:14:15.299947Z",
     "start_time": "2025-02-27T05:14:15.289907Z"
    }
   },
   "cell_type": "code",
   "source": [
    "dsets_index_df = pd.read_csv(dsets_index_path, index_col=\"Idx\")\n",
    "num_dsets = dsets_index_df.shape[0]"
   ],
   "id": "922cfdf6b3489014",
   "outputs": [],
   "execution_count": 3
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-02-27T05:14:16.174093Z",
     "start_time": "2025-02-27T05:14:15.314370Z"
    }
   },
   "cell_type": "code",
   "source": [
    "train_valid_split_specs = load_train_validation_splits()\n",
    "data_selector = ActivationsDataSelector(dsets_index_df, train_valid_split_specs)"
   ],
   "id": "d458a2d34b219fa8",
   "outputs": [],
   "execution_count": 4
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-02-27T05:14:16.189620Z",
     "start_time": "2025-02-27T05:14:16.183130Z"
    }
   },
   "cell_type": "code",
   "source": [
    "logger.info(f\"loading probe-training scenarios from file {train_scenarios_spec_path}\")\n",
    "with train_scenarios_spec_path.open(\"r\") as f:\n",
    "    raw_scenarios_specs = json.load(f)\n",
    "train_scenarios = [ProbeTrainScenario(**raw_scenario_spec) for raw_scenario_spec in raw_scenarios_specs]"
   ],
   "id": "5b262acb87d66d9b",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2025-02-27 00:14:16,185;__main__;INFO:loading probe-training scenarios from file train_scenarios_spec.json\n"
     ]
    }
   ],
   "execution_count": 5
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-02-27T05:14:16.204156Z",
     "start_time": "2025-02-27T05:14:16.199391Z"
    }
   },
   "cell_type": "code",
   "source": [
    "\n",
    "# first level key identifies the scenario (of one or more datasets) which the probe was trained on;\n",
    "# second level key identifies the previously-unseen dataset which the probe is tested on\n",
    "# third level index identifies which train-validation split was used to train a particular group of comparable probes \n",
    "#  for a scenario\n",
    "probes_metrics_on_test_dsets: dict[tuple[int,...], dict[int, list[MetricsForDatasetProbes]]] = {scenario.scenario_key(): {} for scenario in train_scenarios}"
   ],
   "id": "a25dd6c67bda7664",
   "outputs": [],
   "execution_count": 6
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-02-27T05:18:37.712789Z",
     "start_time": "2025-02-27T05:14:16.227679Z"
    }
   },
   "cell_type": "code",
   "source": [
    "for scenario in train_scenarios:\n",
    "    scenario_key = scenario.scenario_key()\n",
    "    curr_scenario_probes: list[ProbesForScenario] = []\n",
    "    for split_variant_idx in range(data_selector.num_split_variants):\n",
    "        trained_probes = load_probes_for_dset(scenario.result_folder_name, scenario.scenario_name, split_variant_idx)\n",
    "        curr_scenario_probes.append(trained_probes)\n",
    "    for test_dset_idx, test_dset_dtls in dsets_index_df.iterrows():\n",
    "        if test_dset_idx in scenario_key:\n",
    "            continue  # it wouldn't be a test-set evaluation if the probe was trained/validated on the same dataset\n",
    "        \n",
    "        probes_metrics_on_test_dsets[scenario_key][test_dset_idx] = []\n",
    "        test_dset = data_selector.grab_all_data_for_dset(test_dset_idx)\n",
    "        for scenario_probes in curr_scenario_probes:\n",
    "            scenario_metrics_on_curr_test_dset = evaluate_classifier_performance(\n",
    "                scenario_probes, test_dset.activations, test_dset.truth_labels)\n",
    "            probes_metrics_on_test_dsets[scenario_key][test_dset_idx].append(scenario_metrics_on_curr_test_dset)"
   ],
   "id": "87d96e14b0006459",
   "outputs": [],
   "execution_count": 7
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-02-27T05:18:37.740562Z",
     "start_time": "2025-02-27T05:18:37.736562Z"
    }
   },
   "cell_type": "code",
   "source": "analysis_results_folder.mkdir(exist_ok=True)",
   "id": "479feb8d6f05647",
   "outputs": [],
   "execution_count": 8
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-02-27T05:18:39.546498Z",
     "start_time": "2025-02-27T05:18:37.768737Z"
    }
   },
   "cell_type": "code",
   "source": [
    "serialized_test_metrics = {\n",
    "    (\" \".join(map(str, scenario_id))): {\n",
    "        str(test_dset_idx): [test_dset_metrics.to_dict() for test_dset_metrics in test_dset_metrics_lst]\n",
    "        for test_dset_idx, test_dset_metrics_lst in test_dsets_metrics.items()\n",
    "    } for scenario_id, test_dsets_metrics in probes_metrics_on_test_dsets.items()\n",
    "}\n",
    "with test_classification_metrics_path.open(\"w\") as f:\n",
    "    json.dump(serialized_test_metrics, f, indent=2)"
   ],
   "id": "c528297bfd145e57",
   "outputs": [],
   "execution_count": 9
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-02-27T05:18:39.553290Z",
     "start_time": "2025-02-27T05:18:39.550523Z"
    }
   },
   "cell_type": "code",
   "source": "",
   "id": "e96c024fe75f6ea3",
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "",
   "id": "6a58231145f6d3da"
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "",
   "id": "4ab4e730ea356426"
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}