eval_inat21_truncate.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "78a40757",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pathlib\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import hier\n",
    "import metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "64fcec72",
   "metadata": {},
   "outputs": [],
   "source": [
    "exp_dirs = {\n",
    "    # lr = 0.1\n",
    "    3: '/mnt/ssd1/projects/2022-01-hierarchical/experiments/2022-04-07-inat21-trunc/flat-depth-3/',\n",
    "    4: '/mnt/ssd1/projects/2022-01-hierarchical/experiments/2022-04-07-inat21-trunc/flat-depth-4/',\n",
    "    5: '/mnt/ssd1/projects/2022-01-hierarchical/experiments/2022-04-07-inat21-trunc/flat-depth-5/',\n",
    "    6: '/mnt/ssd1/projects/2022-01-hierarchical/experiments/2022-04-07-inat21-trunc/flat-depth-6/',\n",
    "    7: '/mnt/ssd1/projects/2022-01-hierarchical/experiments/2022-04-07-inat21-trunc/flat/',\n",
    "\n",
    "#     # lr = 0.01\n",
    "#     3: '/mnt/ssd1/projects/2022-01-hierarchical/experiments/2022-04-07-inat21-trunc/flat-depth-3-lr-0.01/',\n",
    "#     4: '/mnt/ssd1/projects/2022-01-hierarchical/experiments/2022-04-07-inat21-trunc/flat-depth-4-lr-0.01/',\n",
    "#     5: '/mnt/ssd1/projects/2022-01-hierarchical/experiments/2022-04-07-inat21-trunc/flat-depth-5-lr-0.01/',\n",
    "#     6: '/mnt/ssd1/projects/2022-01-hierarchical/experiments/2022-04-07-inat21-trunc/flat-depth-6-lr-0.01/',\n",
    "#     7: '/mnt/ssd1/projects/2022-01-hierarchical/experiments/2022-03-31-inat21mini/flat_softmax-lr-0.01-b-64-wd-0.0003-ep-20',\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "c7012622",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Convert from single array to shards that can be streamed.\n",
    "# For full hierarchy, the probability array is [100000, 10000] > 3GiB.\n",
    "\n",
    "def convert(pkl_file, npz_file):\n",
    "    outputs = np.load(pkl_file, allow_pickle=True)\n",
    "    x = outputs['prob']\n",
    "    # Divide into shards of around 128MiB.\n",
    "    num_shards = np.ceil(np.size(x) * x.dtype.itemsize / (2 ** 27))\n",
    "    np.savez(npz_file, *np.array_split(x, num_shards))\n",
    "\n",
    "for name, exp_dir in exp_dirs.items():\n",
    "    exp_dir = pathlib.Path(exp_dir)\n",
    "    npz_file = exp_dir / 'predictions/prob-epoch-0020.npz'\n",
    "    pkl_file = exp_dir / 'predictions/full-output-epoch-0020.pkl'\n",
    "    if pkl_file.exists() and not npz_file.exists():\n",
    "        convert(pkl_file=pkl_file, npz_file=npz_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "c68da392",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('resources/hierarchy/inat21.csv') as f:\n",
    "    tree, names = hier.make_hierarchy_from_edges(hier.load_edges(f))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "1b56a14b",
   "metadata": {},
   "outputs": [],
   "source": [
    "subtrees = {}\n",
    "node_subsets = {}\n",
    "projections = {}\n",
    "\n",
    "for depth in exp_dirs:\n",
    "    if depth == 7:\n",
    "        subtree = tree\n",
    "        node_subset = np.arange(tree.num_nodes())\n",
    "        projection = np.arange(tree.num_nodes())\n",
    "    else:\n",
    "        subtree_file = f'resources/subtree/inat21_max_depth_{depth}.csv'\n",
    "        with open(subtree_file) as f:\n",
    "            subtree, subtree_names = hier.make_hierarchy_from_edges(hier.load_edges(f))\n",
    "        node_subset = hier.find_subset_index(names, subtree_names)\n",
    "        projection = hier.find_projection(tree, node_subset)\n",
    "\n",
    "    subtrees[depth] = subtree\n",
    "    node_subsets[depth] = node_subset\n",
    "    projections[depth] = projection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "eb17ad28",
   "metadata": {},
   "outputs": [],
   "source": [
    "acc = {}\n",
    "\n",
    "for train_depth in exp_dirs:\n",
    "    output_file = pathlib.Path(exp_dirs[train_depth]) / 'predictions/output-epoch-0020.pkl'\n",
    "    outputs = np.load(output_file, allow_pickle=True)\n",
    "\n",
    "    # To use 'leaf' predictions, we should load the full probability vector\n",
    "    # and do max-leaf inference within the sub-tree.\n",
    "    # On the other hand, 'majority' inference can be projected post-hoc.\n",
    "    pred_subtree = outputs['pred']['majority']\n",
    "    gt_subtree = outputs['gt']\n",
    "    train_node_subset = node_subsets[train_depth]\n",
    "    pred = train_node_subset[pred_subtree]\n",
    "    gt = train_node_subset[gt_subtree]\n",
    "\n",
    "    for test_depth in exp_dirs:\n",
    "        if test_depth > train_depth:\n",
    "            continue\n",
    "        # Project ground-truth to test\n",
    "        project = projections[test_depth]\n",
    "        gt_proj = project[gt]\n",
    "        pred_proj = project[pred]\n",
    "        acc[train_depth, test_depth] = np.mean(gt_proj == pred_proj)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "9931c765",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.78844</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.81713</td>\n",
       "      <td>0.48301</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.83108</td>\n",
       "      <td>0.52361</td>\n",
       "      <td>0.35589</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.83270</td>\n",
       "      <td>0.55167</td>\n",
       "      <td>0.40487</td>\n",
       "      <td>0.26763</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.83447</td>\n",
       "      <td>0.57058</td>\n",
       "      <td>0.42960</td>\n",
       "      <td>0.30600</td>\n",
       "      <td>0.22784</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         3        4        5        6        7\n",
       "3  0.78844      NaN      NaN      NaN      NaN\n",
       "4  0.81713  0.48301      NaN      NaN      NaN\n",
       "5  0.83108  0.52361  0.35589      NaN      NaN\n",
       "6  0.83270  0.55167  0.40487  0.26763      NaN\n",
       "7  0.83447  0.57058  0.42960  0.30600  0.22784"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.Series(acc).unstack()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "6610fa1d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Obtain *per-class* accuracy at each depth.\n",
    "\n",
    "per_class = {}\n",
    "\n",
    "for train_depth in exp_dirs:\n",
    "    output_file = pathlib.Path(exp_dirs[train_depth]) / 'predictions/output-epoch-0020.pkl'\n",
    "    outputs = np.load(output_file, allow_pickle=True)\n",
    "\n",
    "    # To use 'leaf' predictions, we should load the full probability vector\n",
    "    # and do max-leaf inference within the sub-tree.\n",
    "    # On the other hand, 'majority' inference can be projected post-hoc.\n",
    "    pred_subtree = outputs['pred']['majority']\n",
    "    gt_subtree = outputs['gt']\n",
    "    train_node_subset = node_subsets[train_depth]\n",
    "    pred = train_node_subset[pred_subtree]\n",
    "    gt = train_node_subset[gt_subtree]\n",
    "\n",
    "    for test_depth in exp_dirs:\n",
    "        if test_depth > train_depth:\n",
    "            continue\n",
    "        # Project ground-truth to test\n",
    "        project = projections[test_depth]\n",
    "        gt_proj = project[gt]\n",
    "        pred_proj = project[pred]\n",
    "\n",
    "        is_correct = (gt_proj == pred_proj)\n",
    "        per_class[train_depth, test_depth] = np.array([\n",
    "            np.mean(is_correct[gt_proj == y])\n",
    "            for y in subtrees[test_depth].leaf_subset()\n",
    "        ])\n",
    "\n",
    "    del outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "7ad06e48",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style type=\"text/css\">\n",
       "#T_372d1_row0_col0, #T_372d1_row0_col2, #T_372d1_row0_col3, #T_372d1_row0_col4, #T_372d1_row1_col0, #T_372d1_row1_col2, #T_372d1_row1_col3, #T_372d1_row1_col4, #T_372d1_row2_col0, #T_372d1_row2_col2, #T_372d1_row2_col3, #T_372d1_row2_col4, #T_372d1_row2_col5, #T_372d1_row3_col0, #T_372d1_row3_col2, #T_372d1_row3_col3, #T_372d1_row3_col4, #T_372d1_row3_col5, #T_372d1_row4_col0, #T_372d1_row4_col2, #T_372d1_row4_col3, #T_372d1_row5_col0, #T_372d1_row5_col2, #T_372d1_row5_col3, #T_372d1_row6_col0, #T_372d1_row6_col2, #T_372d1_row6_col3, #T_372d1_row6_col4, #T_372d1_row7_col0, #T_372d1_row7_col2, #T_372d1_row7_col3, #T_372d1_row8_col0, #T_372d1_row8_col2, #T_372d1_row8_col3, #T_372d1_row9_col0, #T_372d1_row9_col2, #T_372d1_row9_col3, #T_372d1_row10_col0, #T_372d1_row10_col2, #T_372d1_row10_col3, #T_372d1_row10_col4, #T_372d1_row11_col0, #T_372d1_row11_col2, #T_372d1_row11_col3, #T_372d1_row11_col4, #T_372d1_row12_col0, #T_372d1_row12_col2, #T_372d1_row12_col3, #T_372d1_row12_col4, #T_372d1_row13_col0, #T_372d1_row13_col2, #T_372d1_row13_col3, #T_372d1_row14_col0, #T_372d1_row14_col2, #T_372d1_row14_col3, #T_372d1_row14_col4, #T_372d1_row15_col0, #T_372d1_row15_col2, #T_372d1_row15_col3, #T_372d1_row16_col0, #T_372d1_row16_col2, #T_372d1_row16_col3, #T_372d1_row16_col4, #T_372d1_row17_col0, #T_372d1_row17_col2, #T_372d1_row17_col3, #T_372d1_row17_col4, #T_372d1_row18_col0, #T_372d1_row18_col2, #T_372d1_row18_col3, #T_372d1_row19_col0, #T_372d1_row19_col2, #T_372d1_row19_col3 {\n",
       "  color: black;\n",
       "}\n",
       "#T_372d1_row0_col5, #T_372d1_row0_col6, #T_372d1_row1_col5, #T_372d1_row1_col6, #T_372d1_row2_col6, #T_372d1_row3_col6, #T_372d1_row4_col4, #T_372d1_row4_col5, #T_372d1_row4_col6, #T_372d1_row5_col4, #T_372d1_row5_col5, #T_372d1_row5_col6, #T_372d1_row6_col5, #T_372d1_row6_col6, #T_372d1_row7_col4, #T_372d1_row7_col5, #T_372d1_row7_col6, #T_372d1_row8_col4, #T_372d1_row8_col5, #T_372d1_row8_col6, #T_372d1_row9_col4, #T_372d1_row9_col5, #T_372d1_row9_col6, #T_372d1_row10_col5, #T_372d1_row10_col6, #T_372d1_row11_col5, #T_372d1_row11_col6, #T_372d1_row12_col5, #T_372d1_row12_col6, #T_372d1_row13_col4, #T_372d1_row13_col5, #T_372d1_row13_col6, #T_372d1_row14_col5, #T_372d1_row14_col6, #T_372d1_row15_col4, #T_372d1_row15_col5, #T_372d1_row15_col6, #T_372d1_row16_col5, #T_372d1_row16_col6, #T_372d1_row17_col5, #T_372d1_row17_col6, #T_372d1_row18_col4, #T_372d1_row18_col5, #T_372d1_row18_col6, #T_372d1_row19_col4, #T_372d1_row19_col5, #T_372d1_row19_col6 {\n",
       "  color: green;\n",
       "}\n",
       "</style>\n",
       "<table id=\"T_372d1\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th class=\"blank\" >&nbsp;</th>\n",
       "      <th class=\"blank\" >&nbsp;</th>\n",
       "      <th class=\"blank level0\" >&nbsp;</th>\n",
       "      <th id=\"T_372d1_level0_col0\" class=\"col_heading level0 col0\" >mean</th>\n",
       "      <th id=\"T_372d1_level0_col1\" class=\"col_heading level0 col1\" >std</th>\n",
       "      <th id=\"T_372d1_level0_col2\" class=\"col_heading level0 col2\" >min</th>\n",
       "      <th id=\"T_372d1_level0_col3\" class=\"col_heading level0 col3\" >50%</th>\n",
       "      <th id=\"T_372d1_level0_col4\" class=\"col_heading level0 col4\" >85%</th>\n",
       "      <th id=\"T_372d1_level0_col5\" class=\"col_heading level0 col5\" >95%</th>\n",
       "      <th id=\"T_372d1_level0_col6\" class=\"col_heading level0 col6\" >max</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th class=\"index_name level0\" >test</th>\n",
       "      <th class=\"index_name level1\" >a</th>\n",
       "      <th class=\"index_name level2\" >b</th>\n",
       "      <th class=\"blank col0\" >&nbsp;</th>\n",
       "      <th class=\"blank col1\" >&nbsp;</th>\n",
       "      <th class=\"blank col2\" >&nbsp;</th>\n",
       "      <th class=\"blank col3\" >&nbsp;</th>\n",
       "      <th class=\"blank col4\" >&nbsp;</th>\n",
       "      <th class=\"blank col5\" >&nbsp;</th>\n",
       "      <th class=\"blank col6\" >&nbsp;</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th id=\"T_372d1_level0_row0\" class=\"row_heading level0 row0\" rowspan=\"10\">3</th>\n",
       "      <th id=\"T_372d1_level1_row0\" class=\"row_heading level1 row0\" rowspan=\"4\">3</th>\n",
       "      <th id=\"T_372d1_level2_row0\" class=\"row_heading level2 row0\" >4</th>\n",
       "      <td id=\"T_372d1_row0_col0\" class=\"data row0 col0\" >-5.8%</td>\n",
       "      <td id=\"T_372d1_row0_col1\" class=\"data row0 col1\" >+9.1%</td>\n",
       "      <td id=\"T_372d1_row0_col2\" class=\"data row0 col2\" >-40.0%</td>\n",
       "      <td id=\"T_372d1_row0_col3\" class=\"data row0 col3\" >-3.3%</td>\n",
       "      <td id=\"T_372d1_row0_col4\" class=\"data row0 col4\" >+0.0%</td>\n",
       "      <td id=\"T_372d1_row0_col5\" class=\"data row0 col5\" >+1.5%</td>\n",
       "      <td id=\"T_372d1_row0_col6\" class=\"data row0 col6\" >+16.0%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_372d1_level2_row1\" class=\"row_heading level2 row1\" >5</th>\n",
       "      <td id=\"T_372d1_row1_col0\" class=\"data row1 col0\" >-9.1%</td>\n",
       "      <td id=\"T_372d1_row1_col1\" class=\"data row1 col1\" >+11.5%</td>\n",
       "      <td id=\"T_372d1_row1_col2\" class=\"data row1 col2\" >-45.0%</td>\n",
       "      <td id=\"T_372d1_row1_col3\" class=\"data row1 col3\" >-8.0%</td>\n",
       "      <td id=\"T_372d1_row1_col4\" class=\"data row1 col4\" >+0.0%</td>\n",
       "      <td id=\"T_372d1_row1_col5\" class=\"data row1 col5\" >+2.8%</td>\n",
       "      <td id=\"T_372d1_row1_col6\" class=\"data row1 col6\" >+18.0%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_372d1_level2_row2\" class=\"row_heading level2 row2\" >6</th>\n",
       "      <td id=\"T_372d1_row2_col0\" class=\"data row2 col0\" >-15.7%</td>\n",
       "      <td id=\"T_372d1_row2_col1\" class=\"data row2 col1\" >+15.7%</td>\n",
       "      <td id=\"T_372d1_row2_col2\" class=\"data row2 col2\" >-60.0%</td>\n",
       "      <td id=\"T_372d1_row2_col3\" class=\"data row2 col3\" >-10.0%</td>\n",
       "      <td id=\"T_372d1_row2_col4\" class=\"data row2 col4\" >-2.8%</td>\n",
       "      <td id=\"T_372d1_row2_col5\" class=\"data row2 col5\" >+0.0%</td>\n",
       "      <td id=\"T_372d1_row2_col6\" class=\"data row2 col6\" >+16.0%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_372d1_level2_row3\" class=\"row_heading level2 row3\" >7</th>\n",
       "      <td id=\"T_372d1_row3_col0\" class=\"data row3 col0\" >-18.8%</td>\n",
       "      <td id=\"T_372d1_row3_col1\" class=\"data row3 col1\" >+16.4%</td>\n",
       "      <td id=\"T_372d1_row3_col2\" class=\"data row3 col2\" >-65.0%</td>\n",
       "      <td id=\"T_372d1_row3_col3\" class=\"data row3 col3\" >-15.0%</td>\n",
       "      <td id=\"T_372d1_row3_col4\" class=\"data row3 col4\" >-4.5%</td>\n",
       "      <td id=\"T_372d1_row3_col5\" class=\"data row3 col5\" >-1.6%</td>\n",
       "      <td id=\"T_372d1_row3_col6\" class=\"data row3 col6\" >+12.0%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_372d1_level1_row4\" class=\"row_heading level1 row4\" rowspan=\"3\">4</th>\n",
       "      <th id=\"T_372d1_level2_row4\" class=\"row_heading level2 row4\" >5</th>\n",
       "      <td id=\"T_372d1_row4_col0\" class=\"data row4 col0\" >-3.3%</td>\n",
       "      <td id=\"T_372d1_row4_col1\" class=\"data row4 col1\" >+7.3%</td>\n",
       "      <td id=\"T_372d1_row4_col2\" class=\"data row4 col2\" >-22.0%</td>\n",
       "      <td id=\"T_372d1_row4_col3\" class=\"data row4 col3\" >-1.7%</td>\n",
       "      <td id=\"T_372d1_row4_col4\" class=\"data row4 col4\" >+1.4%</td>\n",
       "      <td id=\"T_372d1_row4_col5\" class=\"data row4 col5\" >+5.3%</td>\n",
       "      <td id=\"T_372d1_row4_col6\" class=\"data row4 col6\" >+16.0%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_372d1_level2_row5\" class=\"row_heading level2 row5\" >6</th>\n",
       "      <td id=\"T_372d1_row5_col0\" class=\"data row5 col0\" >-9.9%</td>\n",
       "      <td id=\"T_372d1_row5_col1\" class=\"data row5 col1\" >+12.3%</td>\n",
       "      <td id=\"T_372d1_row5_col2\" class=\"data row5 col2\" >-50.0%</td>\n",
       "      <td id=\"T_372d1_row5_col3\" class=\"data row5 col3\" >-7.2%</td>\n",
       "      <td id=\"T_372d1_row5_col4\" class=\"data row5 col4\" >+0.5%</td>\n",
       "      <td id=\"T_372d1_row5_col5\" class=\"data row5 col5\" >+4.8%</td>\n",
       "      <td id=\"T_372d1_row5_col6\" class=\"data row5 col6\" >+6.0%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_372d1_level2_row6\" class=\"row_heading level2 row6\" >7</th>\n",
       "      <td id=\"T_372d1_row6_col0\" class=\"data row6 col0\" >-13.0%</td>\n",
       "      <td id=\"T_372d1_row6_col1\" class=\"data row6 col1\" >+13.1%</td>\n",
       "      <td id=\"T_372d1_row6_col2\" class=\"data row6 col2\" >-46.7%</td>\n",
       "      <td id=\"T_372d1_row6_col3\" class=\"data row6 col3\" >-10.0%</td>\n",
       "      <td id=\"T_372d1_row6_col4\" class=\"data row6 col4\" >-0.2%</td>\n",
       "      <td id=\"T_372d1_row6_col5\" class=\"data row6 col5\" >+2.7%</td>\n",
       "      <td id=\"T_372d1_row6_col6\" class=\"data row6 col6\" >+3.6%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_372d1_level1_row7\" class=\"row_heading level1 row7\" rowspan=\"2\">5</th>\n",
       "      <th id=\"T_372d1_level2_row7\" class=\"row_heading level2 row7\" >6</th>\n",
       "      <td id=\"T_372d1_row7_col0\" class=\"data row7 col0\" >-6.6%</td>\n",
       "      <td id=\"T_372d1_row7_col1\" class=\"data row7 col1\" >+9.0%</td>\n",
       "      <td id=\"T_372d1_row7_col2\" class=\"data row7 col2\" >-40.0%</td>\n",
       "      <td id=\"T_372d1_row7_col3\" class=\"data row7 col3\" >-5.7%</td>\n",
       "      <td id=\"T_372d1_row7_col4\" class=\"data row7 col4\" >+0.5%</td>\n",
       "      <td id=\"T_372d1_row7_col5\" class=\"data row7 col5\" >+4.0%</td>\n",
       "      <td id=\"T_372d1_row7_col6\" class=\"data row7 col6\" >+10.0%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_372d1_level2_row8\" class=\"row_heading level2 row8\" >7</th>\n",
       "      <td id=\"T_372d1_row8_col0\" class=\"data row8 col0\" >-9.7%</td>\n",
       "      <td id=\"T_372d1_row8_col1\" class=\"data row8 col1\" >+10.2%</td>\n",
       "      <td id=\"T_372d1_row8_col2\" class=\"data row8 col2\" >-40.0%</td>\n",
       "      <td id=\"T_372d1_row8_col3\" class=\"data row8 col3\" >-10.0%</td>\n",
       "      <td id=\"T_372d1_row8_col4\" class=\"data row8 col4\" >+0.0%</td>\n",
       "      <td id=\"T_372d1_row8_col5\" class=\"data row8 col5\" >+4.1%</td>\n",
       "      <td id=\"T_372d1_row8_col6\" class=\"data row8 col6\" >+10.0%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_372d1_level1_row9\" class=\"row_heading level1 row9\" >6</th>\n",
       "      <th id=\"T_372d1_level2_row9\" class=\"row_heading level2 row9\" >7</th>\n",
       "      <td id=\"T_372d1_row9_col0\" class=\"data row9 col0\" >-3.1%</td>\n",
       "      <td id=\"T_372d1_row9_col1\" class=\"data row9 col1\" >+10.5%</td>\n",
       "      <td id=\"T_372d1_row9_col2\" class=\"data row9 col2\" >-30.0%</td>\n",
       "      <td id=\"T_372d1_row9_col3\" class=\"data row9 col3\" >-2.1%</td>\n",
       "      <td id=\"T_372d1_row9_col4\" class=\"data row9 col4\" >+0.7%</td>\n",
       "      <td id=\"T_372d1_row9_col5\" class=\"data row9 col5\" >+8.8%</td>\n",
       "      <td id=\"T_372d1_row9_col6\" class=\"data row9 col6\" >+50.0%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_372d1_level0_row10\" class=\"row_heading level0 row10\" rowspan=\"6\">4</th>\n",
       "      <th id=\"T_372d1_level1_row10\" class=\"row_heading level1 row10\" rowspan=\"3\">4</th>\n",
       "      <th id=\"T_372d1_level2_row10\" class=\"row_heading level2 row10\" >5</th>\n",
       "      <td id=\"T_372d1_row10_col0\" class=\"data row10 col0\" >-6.3%</td>\n",
       "      <td id=\"T_372d1_row10_col1\" class=\"data row10 col1\" >+10.2%</td>\n",
       "      <td id=\"T_372d1_row10_col2\" class=\"data row10 col2\" >-50.0%</td>\n",
       "      <td id=\"T_372d1_row10_col3\" class=\"data row10 col3\" >-4.1%</td>\n",
       "      <td id=\"T_372d1_row10_col4\" class=\"data row10 col4\" >+0.0%</td>\n",
       "      <td id=\"T_372d1_row10_col5\" class=\"data row10 col5\" >+3.0%</td>\n",
       "      <td id=\"T_372d1_row10_col6\" class=\"data row10 col6\" >+32.5%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_372d1_level2_row11\" class=\"row_heading level2 row11\" >6</th>\n",
       "      <td id=\"T_372d1_row11_col0\" class=\"data row11 col0\" >-13.3%</td>\n",
       "      <td id=\"T_372d1_row11_col1\" class=\"data row11 col1\" >+15.1%</td>\n",
       "      <td id=\"T_372d1_row11_col2\" class=\"data row11 col2\" >-80.0%</td>\n",
       "      <td id=\"T_372d1_row11_col3\" class=\"data row11 col3\" >-10.0%</td>\n",
       "      <td id=\"T_372d1_row11_col4\" class=\"data row11 col4\" >+0.0%</td>\n",
       "      <td id=\"T_372d1_row11_col5\" class=\"data row11 col5\" >+3.3%</td>\n",
       "      <td id=\"T_372d1_row11_col6\" class=\"data row11 col6\" >+12.5%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_372d1_level2_row12\" class=\"row_heading level2 row12\" >7</th>\n",
       "      <td id=\"T_372d1_row12_col0\" class=\"data row12 col0\" >-16.9%</td>\n",
       "      <td id=\"T_372d1_row12_col1\" class=\"data row12 col1\" >+16.3%</td>\n",
       "      <td id=\"T_372d1_row12_col2\" class=\"data row12 col2\" >-90.0%</td>\n",
       "      <td id=\"T_372d1_row12_col3\" class=\"data row12 col3\" >-15.0%</td>\n",
       "      <td id=\"T_372d1_row12_col4\" class=\"data row12 col4\" >+0.0%</td>\n",
       "      <td id=\"T_372d1_row12_col5\" class=\"data row12 col5\" >+1.8%</td>\n",
       "      <td id=\"T_372d1_row12_col6\" class=\"data row12 col6\" >+12.5%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_372d1_level1_row13\" class=\"row_heading level1 row13\" rowspan=\"2\">5</th>\n",
       "      <th id=\"T_372d1_level2_row13\" class=\"row_heading level2 row13\" >6</th>\n",
       "      <td id=\"T_372d1_row13_col0\" class=\"data row13 col0\" >-7.1%</td>\n",
       "      <td id=\"T_372d1_row13_col1\" class=\"data row13 col1\" >+11.1%</td>\n",
       "      <td id=\"T_372d1_row13_col2\" class=\"data row13 col2\" >-70.0%</td>\n",
       "      <td id=\"T_372d1_row13_col3\" class=\"data row13 col3\" >-5.0%</td>\n",
       "      <td id=\"T_372d1_row13_col4\" class=\"data row13 col4\" >+1.7%</td>\n",
       "      <td id=\"T_372d1_row13_col5\" class=\"data row13 col5\" >+6.8%</td>\n",
       "      <td id=\"T_372d1_row13_col6\" class=\"data row13 col6\" >+12.9%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_372d1_level2_row14\" class=\"row_heading level2 row14\" >7</th>\n",
       "      <td id=\"T_372d1_row14_col0\" class=\"data row14 col0\" >-10.6%</td>\n",
       "      <td id=\"T_372d1_row14_col1\" class=\"data row14 col1\" >+12.5%</td>\n",
       "      <td id=\"T_372d1_row14_col2\" class=\"data row14 col2\" >-50.0%</td>\n",
       "      <td id=\"T_372d1_row14_col3\" class=\"data row14 col3\" >-10.0%</td>\n",
       "      <td id=\"T_372d1_row14_col4\" class=\"data row14 col4\" >+0.0%</td>\n",
       "      <td id=\"T_372d1_row14_col5\" class=\"data row14 col5\" >+6.1%</td>\n",
       "      <td id=\"T_372d1_row14_col6\" class=\"data row14 col6\" >+11.4%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_372d1_level1_row15\" class=\"row_heading level1 row15\" >6</th>\n",
       "      <th id=\"T_372d1_level2_row15\" class=\"row_heading level2 row15\" >7</th>\n",
       "      <td id=\"T_372d1_row15_col0\" class=\"data row15 col0\" >-3.6%</td>\n",
       "      <td id=\"T_372d1_row15_col1\" class=\"data row15 col1\" >+8.6%</td>\n",
       "      <td id=\"T_372d1_row15_col2\" class=\"data row15 col2\" >-50.0%</td>\n",
       "      <td id=\"T_372d1_row15_col3\" class=\"data row15 col3\" >-3.2%</td>\n",
       "      <td id=\"T_372d1_row15_col4\" class=\"data row15 col4\" >+1.7%</td>\n",
       "      <td id=\"T_372d1_row15_col5\" class=\"data row15 col5\" >+10.0%</td>\n",
       "      <td id=\"T_372d1_row15_col6\" class=\"data row15 col6\" >+50.0%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_372d1_level0_row16\" class=\"row_heading level0 row16\" rowspan=\"3\">5</th>\n",
       "      <th id=\"T_372d1_level1_row16\" class=\"row_heading level1 row16\" rowspan=\"2\">5</th>\n",
       "      <th id=\"T_372d1_level2_row16\" class=\"row_heading level2 row16\" >6</th>\n",
       "      <td id=\"T_372d1_row16_col0\" class=\"data row16 col0\" >-8.6%</td>\n",
       "      <td id=\"T_372d1_row16_col1\" class=\"data row16 col1\" >+12.0%</td>\n",
       "      <td id=\"T_372d1_row16_col2\" class=\"data row16 col2\" >-70.0%</td>\n",
       "      <td id=\"T_372d1_row16_col3\" class=\"data row16 col3\" >-5.9%</td>\n",
       "      <td id=\"T_372d1_row16_col4\" class=\"data row16 col4\" >+0.0%</td>\n",
       "      <td id=\"T_372d1_row16_col5\" class=\"data row16 col5\" >+5.0%</td>\n",
       "      <td id=\"T_372d1_row16_col6\" class=\"data row16 col6\" >+25.0%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_372d1_level2_row17\" class=\"row_heading level2 row17\" >7</th>\n",
       "      <td id=\"T_372d1_row17_col0\" class=\"data row17 col0\" >-12.6%</td>\n",
       "      <td id=\"T_372d1_row17_col1\" class=\"data row17 col1\" >+14.1%</td>\n",
       "      <td id=\"T_372d1_row17_col2\" class=\"data row17 col2\" >-80.0%</td>\n",
       "      <td id=\"T_372d1_row17_col3\" class=\"data row17 col3\" >-10.0%</td>\n",
       "      <td id=\"T_372d1_row17_col4\" class=\"data row17 col4\" >+0.0%</td>\n",
       "      <td id=\"T_372d1_row17_col5\" class=\"data row17 col5\" >+5.0%</td>\n",
       "      <td id=\"T_372d1_row17_col6\" class=\"data row17 col6\" >+25.0%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_372d1_level1_row18\" class=\"row_heading level1 row18\" >6</th>\n",
       "      <th id=\"T_372d1_level2_row18\" class=\"row_heading level2 row18\" >7</th>\n",
       "      <td id=\"T_372d1_row18_col0\" class=\"data row18 col0\" >-4.0%</td>\n",
       "      <td id=\"T_372d1_row18_col1\" class=\"data row18 col1\" >+9.1%</td>\n",
       "      <td id=\"T_372d1_row18_col2\" class=\"data row18 col2\" >-50.0%</td>\n",
       "      <td id=\"T_372d1_row18_col3\" class=\"data row18 col3\" >-3.2%</td>\n",
       "      <td id=\"T_372d1_row18_col4\" class=\"data row18 col4\" >+2.4%</td>\n",
       "      <td id=\"T_372d1_row18_col5\" class=\"data row18 col5\" >+10.0%</td>\n",
       "      <td id=\"T_372d1_row18_col6\" class=\"data row18 col6\" >+50.0%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_372d1_level0_row19\" class=\"row_heading level0 row19\" >6</th>\n",
       "      <th id=\"T_372d1_level1_row19\" class=\"row_heading level1 row19\" >6</th>\n",
       "      <th id=\"T_372d1_level2_row19\" class=\"row_heading level2 row19\" >7</th>\n",
       "      <td id=\"T_372d1_row19_col0\" class=\"data row19 col0\" >-5.1%</td>\n",
       "      <td id=\"T_372d1_row19_col1\" class=\"data row19 col1\" >+11.2%</td>\n",
       "      <td id=\"T_372d1_row19_col2\" class=\"data row19 col2\" >-70.0%</td>\n",
       "      <td id=\"T_372d1_row19_col3\" class=\"data row19 col3\" >+0.0%</td>\n",
       "      <td id=\"T_372d1_row19_col4\" class=\"data row19 col4\" >+3.3%</td>\n",
       "      <td id=\"T_372d1_row19_col5\" class=\"data row19 col5\" >+10.0%</td>\n",
       "      <td id=\"T_372d1_row19_col6\" class=\"data row19 col6\" >+50.0%</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7f6f11f4b880>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check whether it is ever advantageous to train at a *coarser* level.\n",
    "# Consider training at two different levels A < B (i.e. B is deeper).\n",
    "# Compare the per-class accuracy at some level <= A.\n",
    "\n",
    "# Compute (level A) - (level B).\n",
    "# Greater positive numbers mean that coarse training is better.\n",
    "percentiles = [0.5, 0.85, 0.95]\n",
    "\n",
    "rows = {}\n",
    "\n",
    "for depth_a in exp_dirs:\n",
    "    for depth_b in exp_dirs:\n",
    "        if not depth_a < depth_b:\n",
    "            continue\n",
    "        for test_depth in exp_dirs:\n",
    "            if not test_depth <= depth_a:\n",
    "                continue\n",
    "            rows[test_depth, depth_a, depth_b] = pd.DataFrame([\n",
    "                per_class[depth_a, test_depth] - per_class[depth_b, test_depth]\n",
    "            ]).T.describe(percentiles).T\n",
    "\n",
    "table = pd.concat(rows).sort_index()\n",
    "table = table.droplevel(3)\n",
    "table.index = table.index.set_names(['test', 'a', 'b'])\n",
    "del table['count']\n",
    "# table.format('{:+.1%}')\n",
    "\n",
    "def highlight(s):\n",
    "    if s.name == 'std':\n",
    "        return None\n",
    "    return ['color: green;' if x > 0 else 'color: black;' for x in s]\n",
    "\n",
    "table.style.apply(highlight).format('{:+.1%}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "5b7719e1",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style type=\"text/css\">\n",
       "#T_3e542_row0_col0, #T_3e542_row0_col2, #T_3e542_row0_col3, #T_3e542_row0_col4, #T_3e542_row1_col0, #T_3e542_row1_col2, #T_3e542_row1_col3, #T_3e542_row1_col4, #T_3e542_row2_col0, #T_3e542_row2_col2, #T_3e542_row2_col3, #T_3e542_row2_col4, #T_3e542_row2_col5, #T_3e542_row3_col0, #T_3e542_row3_col2, #T_3e542_row3_col3, #T_3e542_row3_col4, #T_3e542_row3_col5, #T_3e542_row4_col0, #T_3e542_row4_col2, #T_3e542_row4_col3, #T_3e542_row4_col4, #T_3e542_row5_col0, #T_3e542_row5_col2, #T_3e542_row5_col3, #T_3e542_row5_col4, #T_3e542_row6_col0, #T_3e542_row6_col2, #T_3e542_row6_col3, #T_3e542_row6_col4, #T_3e542_row7_col0, #T_3e542_row7_col2, #T_3e542_row7_col3, #T_3e542_row7_col4, #T_3e542_row8_col0, #T_3e542_row8_col2, #T_3e542_row8_col3, #T_3e542_row8_col4, #T_3e542_row9_col0, #T_3e542_row9_col2, #T_3e542_row9_col3 {\n",
       "  color: black;\n",
       "}\n",
       "#T_3e542_row0_col5, #T_3e542_row0_col6, #T_3e542_row1_col5, #T_3e542_row1_col6, #T_3e542_row2_col6, #T_3e542_row3_col6, #T_3e542_row4_col5, #T_3e542_row4_col6, #T_3e542_row5_col5, #T_3e542_row5_col6, #T_3e542_row6_col5, #T_3e542_row6_col6, #T_3e542_row7_col5, #T_3e542_row7_col6, #T_3e542_row8_col5, #T_3e542_row8_col6, #T_3e542_row9_col4, #T_3e542_row9_col5, #T_3e542_row9_col6 {\n",
       "  color: green;\n",
       "}\n",
       "</style>\n",
       "<table id=\"T_3e542\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th class=\"blank\" >&nbsp;</th>\n",
       "      <th class=\"blank level0\" >&nbsp;</th>\n",
       "      <th id=\"T_3e542_level0_col0\" class=\"col_heading level0 col0\" >mean</th>\n",
       "      <th id=\"T_3e542_level0_col1\" class=\"col_heading level0 col1\" >std</th>\n",
       "      <th id=\"T_3e542_level0_col2\" class=\"col_heading level0 col2\" >min</th>\n",
       "      <th id=\"T_3e542_level0_col3\" class=\"col_heading level0 col3\" >50%</th>\n",
       "      <th id=\"T_3e542_level0_col4\" class=\"col_heading level0 col4\" >85%</th>\n",
       "      <th id=\"T_3e542_level0_col5\" class=\"col_heading level0 col5\" >95%</th>\n",
       "      <th id=\"T_3e542_level0_col6\" class=\"col_heading level0 col6\" >max</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th class=\"index_name level0\" >test</th>\n",
       "      <th class=\"index_name level1\" >train</th>\n",
       "      <th class=\"blank col0\" >&nbsp;</th>\n",
       "      <th class=\"blank col1\" >&nbsp;</th>\n",
       "      <th class=\"blank col2\" >&nbsp;</th>\n",
       "      <th class=\"blank col3\" >&nbsp;</th>\n",
       "      <th class=\"blank col4\" >&nbsp;</th>\n",
       "      <th class=\"blank col5\" >&nbsp;</th>\n",
       "      <th class=\"blank col6\" >&nbsp;</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th id=\"T_3e542_level0_row0\" class=\"row_heading level0 row0\" rowspan=\"4\">3</th>\n",
       "      <th id=\"T_3e542_level1_row0\" class=\"row_heading level1 row0\" >4</th>\n",
       "      <td id=\"T_3e542_row0_col0\" class=\"data row0 col0\" >-5.8%</td>\n",
       "      <td id=\"T_3e542_row0_col1\" class=\"data row0 col1\" >+9.1%</td>\n",
       "      <td id=\"T_3e542_row0_col2\" class=\"data row0 col2\" >-40.0%</td>\n",
       "      <td id=\"T_3e542_row0_col3\" class=\"data row0 col3\" >-3.3%</td>\n",
       "      <td id=\"T_3e542_row0_col4\" class=\"data row0 col4\" >+0.0%</td>\n",
       "      <td id=\"T_3e542_row0_col5\" class=\"data row0 col5\" >+1.5%</td>\n",
       "      <td id=\"T_3e542_row0_col6\" class=\"data row0 col6\" >+16.0%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_3e542_level1_row1\" class=\"row_heading level1 row1\" >5</th>\n",
       "      <td id=\"T_3e542_row1_col0\" class=\"data row1 col0\" >-9.1%</td>\n",
       "      <td id=\"T_3e542_row1_col1\" class=\"data row1 col1\" >+11.5%</td>\n",
       "      <td id=\"T_3e542_row1_col2\" class=\"data row1 col2\" >-45.0%</td>\n",
       "      <td id=\"T_3e542_row1_col3\" class=\"data row1 col3\" >-8.0%</td>\n",
       "      <td id=\"T_3e542_row1_col4\" class=\"data row1 col4\" >+0.0%</td>\n",
       "      <td id=\"T_3e542_row1_col5\" class=\"data row1 col5\" >+2.8%</td>\n",
       "      <td id=\"T_3e542_row1_col6\" class=\"data row1 col6\" >+18.0%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_3e542_level1_row2\" class=\"row_heading level1 row2\" >6</th>\n",
       "      <td id=\"T_3e542_row2_col0\" class=\"data row2 col0\" >-15.7%</td>\n",
       "      <td id=\"T_3e542_row2_col1\" class=\"data row2 col1\" >+15.7%</td>\n",
       "      <td id=\"T_3e542_row2_col2\" class=\"data row2 col2\" >-60.0%</td>\n",
       "      <td id=\"T_3e542_row2_col3\" class=\"data row2 col3\" >-10.0%</td>\n",
       "      <td id=\"T_3e542_row2_col4\" class=\"data row2 col4\" >-2.8%</td>\n",
       "      <td id=\"T_3e542_row2_col5\" class=\"data row2 col5\" >+0.0%</td>\n",
       "      <td id=\"T_3e542_row2_col6\" class=\"data row2 col6\" >+16.0%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_3e542_level1_row3\" class=\"row_heading level1 row3\" >7</th>\n",
       "      <td id=\"T_3e542_row3_col0\" class=\"data row3 col0\" >-18.8%</td>\n",
       "      <td id=\"T_3e542_row3_col1\" class=\"data row3 col1\" >+16.4%</td>\n",
       "      <td id=\"T_3e542_row3_col2\" class=\"data row3 col2\" >-65.0%</td>\n",
       "      <td id=\"T_3e542_row3_col3\" class=\"data row3 col3\" >-15.0%</td>\n",
       "      <td id=\"T_3e542_row3_col4\" class=\"data row3 col4\" >-4.5%</td>\n",
       "      <td id=\"T_3e542_row3_col5\" class=\"data row3 col5\" >-1.6%</td>\n",
       "      <td id=\"T_3e542_row3_col6\" class=\"data row3 col6\" >+12.0%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_3e542_level0_row4\" class=\"row_heading level0 row4\" rowspan=\"3\">4</th>\n",
       "      <th id=\"T_3e542_level1_row4\" class=\"row_heading level1 row4\" >5</th>\n",
       "      <td id=\"T_3e542_row4_col0\" class=\"data row4 col0\" >-6.3%</td>\n",
       "      <td id=\"T_3e542_row4_col1\" class=\"data row4 col1\" >+10.2%</td>\n",
       "      <td id=\"T_3e542_row4_col2\" class=\"data row4 col2\" >-50.0%</td>\n",
       "      <td id=\"T_3e542_row4_col3\" class=\"data row4 col3\" >-4.1%</td>\n",
       "      <td id=\"T_3e542_row4_col4\" class=\"data row4 col4\" >+0.0%</td>\n",
       "      <td id=\"T_3e542_row4_col5\" class=\"data row4 col5\" >+3.0%</td>\n",
       "      <td id=\"T_3e542_row4_col6\" class=\"data row4 col6\" >+32.5%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_3e542_level1_row5\" class=\"row_heading level1 row5\" >6</th>\n",
       "      <td id=\"T_3e542_row5_col0\" class=\"data row5 col0\" >-13.3%</td>\n",
       "      <td id=\"T_3e542_row5_col1\" class=\"data row5 col1\" >+15.1%</td>\n",
       "      <td id=\"T_3e542_row5_col2\" class=\"data row5 col2\" >-80.0%</td>\n",
       "      <td id=\"T_3e542_row5_col3\" class=\"data row5 col3\" >-10.0%</td>\n",
       "      <td id=\"T_3e542_row5_col4\" class=\"data row5 col4\" >+0.0%</td>\n",
       "      <td id=\"T_3e542_row5_col5\" class=\"data row5 col5\" >+3.3%</td>\n",
       "      <td id=\"T_3e542_row5_col6\" class=\"data row5 col6\" >+12.5%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_3e542_level1_row6\" class=\"row_heading level1 row6\" >7</th>\n",
       "      <td id=\"T_3e542_row6_col0\" class=\"data row6 col0\" >-16.9%</td>\n",
       "      <td id=\"T_3e542_row6_col1\" class=\"data row6 col1\" >+16.3%</td>\n",
       "      <td id=\"T_3e542_row6_col2\" class=\"data row6 col2\" >-90.0%</td>\n",
       "      <td id=\"T_3e542_row6_col3\" class=\"data row6 col3\" >-15.0%</td>\n",
       "      <td id=\"T_3e542_row6_col4\" class=\"data row6 col4\" >+0.0%</td>\n",
       "      <td id=\"T_3e542_row6_col5\" class=\"data row6 col5\" >+1.8%</td>\n",
       "      <td id=\"T_3e542_row6_col6\" class=\"data row6 col6\" >+12.5%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_3e542_level0_row7\" class=\"row_heading level0 row7\" rowspan=\"2\">5</th>\n",
       "      <th id=\"T_3e542_level1_row7\" class=\"row_heading level1 row7\" >6</th>\n",
       "      <td id=\"T_3e542_row7_col0\" class=\"data row7 col0\" >-8.6%</td>\n",
       "      <td id=\"T_3e542_row7_col1\" class=\"data row7 col1\" >+12.0%</td>\n",
       "      <td id=\"T_3e542_row7_col2\" class=\"data row7 col2\" >-70.0%</td>\n",
       "      <td id=\"T_3e542_row7_col3\" class=\"data row7 col3\" >-5.9%</td>\n",
       "      <td id=\"T_3e542_row7_col4\" class=\"data row7 col4\" >+0.0%</td>\n",
       "      <td id=\"T_3e542_row7_col5\" class=\"data row7 col5\" >+5.0%</td>\n",
       "      <td id=\"T_3e542_row7_col6\" class=\"data row7 col6\" >+25.0%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_3e542_level1_row8\" class=\"row_heading level1 row8\" >7</th>\n",
       "      <td id=\"T_3e542_row8_col0\" class=\"data row8 col0\" >-12.6%</td>\n",
       "      <td id=\"T_3e542_row8_col1\" class=\"data row8 col1\" >+14.1%</td>\n",
       "      <td id=\"T_3e542_row8_col2\" class=\"data row8 col2\" >-80.0%</td>\n",
       "      <td id=\"T_3e542_row8_col3\" class=\"data row8 col3\" >-10.0%</td>\n",
       "      <td id=\"T_3e542_row8_col4\" class=\"data row8 col4\" >+0.0%</td>\n",
       "      <td id=\"T_3e542_row8_col5\" class=\"data row8 col5\" >+5.0%</td>\n",
       "      <td id=\"T_3e542_row8_col6\" class=\"data row8 col6\" >+25.0%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_3e542_level0_row9\" class=\"row_heading level0 row9\" >6</th>\n",
       "      <th id=\"T_3e542_level1_row9\" class=\"row_heading level1 row9\" >7</th>\n",
       "      <td id=\"T_3e542_row9_col0\" class=\"data row9 col0\" >-5.1%</td>\n",
       "      <td id=\"T_3e542_row9_col1\" class=\"data row9 col1\" >+11.2%</td>\n",
       "      <td id=\"T_3e542_row9_col2\" class=\"data row9 col2\" >-70.0%</td>\n",
       "      <td id=\"T_3e542_row9_col3\" class=\"data row9 col3\" >+0.0%</td>\n",
       "      <td id=\"T_3e542_row9_col4\" class=\"data row9 col4\" >+3.3%</td>\n",
       "      <td id=\"T_3e542_row9_col5\" class=\"data row9 col5\" >+10.0%</td>\n",
       "      <td id=\"T_3e542_row9_col6\" class=\"data row9 col6\" >+50.0%</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7f6f11bf9a30>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Same thing but always evaluate at coarse level.\n",
    "# (This is just less information to interpret.)\n",
    "# Greater positive numbers mean coarser is better.\n",
    "\n",
    "percentiles = [0.5, 0.85, 0.95]\n",
    "\n",
    "rows = {}\n",
    "for test_depth in exp_dirs:\n",
    "    for train_depth in exp_dirs:\n",
    "        if train_depth <= test_depth:\n",
    "            continue\n",
    "        rows[test_depth, train_depth] = pd.DataFrame(\n",
    "            [per_class[test_depth, test_depth] - per_class[train_depth, test_depth]]\n",
    "        ).T.describe(percentiles).T\n",
    "\n",
    "table = pd.concat(rows)\n",
    "table = table.droplevel(2)\n",
    "table.index = table.index.set_names(['test', 'train'])\n",
    "del table['count']\n",
    "# table.format('{:+.1%}')\n",
    "\n",
    "def highlight(s):\n",
    "    if s.name == 'std':\n",
    "        return None\n",
    "    return ['color: green;' if x > 0 else 'color: black;' for x in s]\n",
    "\n",
    "table.style.apply(highlight).format('{:+.1%}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "f8a88091",
   "metadata": {},
   "outputs": [],
   "source": [
    "import infer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "d4cae85d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n"
     ]
    }
   ],
   "source": [
    "acc = {}\n",
    "\n",
    "for train_depth in exp_dirs:\n",
    "    print(train_depth)\n",
    "    output_file = pathlib.Path(exp_dirs[train_depth]) / 'predictions/output-epoch-0020.pkl'\n",
    "    outputs = np.load(output_file, allow_pickle=True)\n",
    "    prob_file = pathlib.Path(exp_dirs[train_depth]) / 'predictions/prob-epoch-0020.npz'\n",
    "    prob = np.load(prob_file, allow_pickle=False)\n",
    "\n",
    "    train_subset = node_subsets[train_depth]\n",
    "    train_projection = projections[train_depth]\n",
    "    gt_train = train_subset[outputs['gt']]\n",
    "\n",
    "    for test_depth in exp_dirs:\n",
    "        if test_depth > train_depth:\n",
    "            continue\n",
    "        test_subset = node_subsets[test_depth]\n",
    "        test_projection = projections[test_depth]\n",
    "        \n",
    "        is_leaf = subtrees[test_depth].leaf_mask()\n",
    "\n",
    "        pred = []\n",
    "        for i, prob_i in enumerate(prob.values()):\n",
    "            pred_i = infer.argmax_where(prob_i[:, train_projection[test_subset]], is_leaf)\n",
    "            pred.append(pred_i)\n",
    "        pred = np.concatenate(pred, axis=0)\n",
    "\n",
    "        gt = test_projection[gt_train]\n",
    "        acc[train_depth, test_depth] = np.mean(gt == pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "08af162f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.82732</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.85242</td>\n",
       "      <td>0.60735</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.86614</td>\n",
       "      <td>0.64661</td>\n",
       "      <td>0.50681</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.86995</td>\n",
       "      <td>0.68298</td>\n",
       "      <td>0.56823</td>\n",
       "      <td>0.44626</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.87368</td>\n",
       "      <td>0.70484</td>\n",
       "      <td>0.59800</td>\n",
       "      <td>0.49295</td>\n",
       "      <td>0.40945</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         3        4        5        6        7\n",
       "3  0.82732      NaN      NaN      NaN      NaN\n",
       "4  0.85242  0.60735      NaN      NaN      NaN\n",
       "5  0.86614  0.64661  0.50681      NaN      NaN\n",
       "6  0.86995  0.68298  0.56823  0.44626      NaN\n",
       "7  0.87368  0.70484  0.59800  0.49295  0.40945"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.Series(acc).unstack()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "96f51afd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n"
     ]
    }
   ],
   "source": [
    "# Obtain *per-class* accuracy at each depth.\n",
    "\n",
    "per_class = {}\n",
    "\n",
    "for train_depth in exp_dirs:\n",
    "    print(train_depth)\n",
    "    output_file = pathlib.Path(exp_dirs[train_depth]) / 'predictions/output-epoch-0020.pkl'\n",
    "    outputs = np.load(output_file, allow_pickle=True)\n",
    "    prob_file = pathlib.Path(exp_dirs[train_depth]) / 'predictions/prob-epoch-0020.npz'\n",
    "    prob = np.load(prob_file, allow_pickle=False)\n",
    "\n",
    "    train_subset = node_subsets[train_depth]\n",
    "    train_projection = projections[train_depth]\n",
    "    gt_train = train_subset[outputs['gt']]\n",
    "\n",
    "    for test_depth in exp_dirs:\n",
    "        if test_depth > train_depth:\n",
    "            continue\n",
    "        test_subset = node_subsets[test_depth]\n",
    "        test_projection = projections[test_depth]\n",
    "        \n",
    "        is_leaf = subtrees[test_depth].leaf_mask()\n",
    "\n",
    "        pred = []\n",
    "        for i, prob_i in enumerate(prob.values()):\n",
    "            pred_i = infer.argmax_where(prob_i[:, train_projection[test_subset]], is_leaf)\n",
    "            pred.append(pred_i)\n",
    "        pred = np.concatenate(pred, axis=0)\n",
    "\n",
    "        gt = test_projection[gt_train]\n",
    "\n",
    "        is_correct = (gt == pred)\n",
    "        per_class[train_depth, test_depth] = np.array([\n",
    "            np.mean(is_correct[gt == y])\n",
    "            for y in subtrees[test_depth].leaf_subset()\n",
    "        ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "ed1ba24d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style type=\"text/css\">\n",
       "</style>\n",
       "<table id=\"T_61f29\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th class=\"blank\" >&nbsp;</th>\n",
       "      <th class=\"blank level0\" >&nbsp;</th>\n",
       "      <th id=\"T_61f29_level0_col0\" class=\"col_heading level0 col0\" >mean</th>\n",
       "      <th id=\"T_61f29_level0_col1\" class=\"col_heading level0 col1\" >std</th>\n",
       "      <th id=\"T_61f29_level0_col2\" class=\"col_heading level0 col2\" >min</th>\n",
       "      <th id=\"T_61f29_level0_col3\" class=\"col_heading level0 col3\" >25%</th>\n",
       "      <th id=\"T_61f29_level0_col4\" class=\"col_heading level0 col4\" >50%</th>\n",
       "      <th id=\"T_61f29_level0_col5\" class=\"col_heading level0 col5\" >75%</th>\n",
       "      <th id=\"T_61f29_level0_col6\" class=\"col_heading level0 col6\" >max</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th class=\"index_name level0\" >test</th>\n",
       "      <th class=\"index_name level1\" >train</th>\n",
       "      <th class=\"blank col0\" >&nbsp;</th>\n",
       "      <th class=\"blank col1\" >&nbsp;</th>\n",
       "      <th class=\"blank col2\" >&nbsp;</th>\n",
       "      <th class=\"blank col3\" >&nbsp;</th>\n",
       "      <th class=\"blank col4\" >&nbsp;</th>\n",
       "      <th class=\"blank col5\" >&nbsp;</th>\n",
       "      <th class=\"blank col6\" >&nbsp;</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th id=\"T_61f29_level0_row0\" class=\"row_heading level0 row0\" rowspan=\"5\">3</th>\n",
       "      <th id=\"T_61f29_level1_row0\" class=\"row_heading level1 row0\" >3</th>\n",
       "      <td id=\"T_61f29_row0_col0\" class=\"data row0 col0\" >32.7%</td>\n",
       "      <td id=\"T_61f29_row0_col1\" class=\"data row0 col1\" >30.3%</td>\n",
       "      <td id=\"T_61f29_row0_col2\" class=\"data row0 col2\" >0.0%</td>\n",
       "      <td id=\"T_61f29_row0_col3\" class=\"data row0 col3\" >0.0%</td>\n",
       "      <td id=\"T_61f29_row0_col4\" class=\"data row0 col4\" >31.1%</td>\n",
       "      <td id=\"T_61f29_row0_col5\" class=\"data row0 col5\" >54.4%</td>\n",
       "      <td id=\"T_61f29_row0_col6\" class=\"data row0 col6\" >92.2%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_61f29_level1_row1\" class=\"row_heading level1 row1\" >4</th>\n",
       "      <td id=\"T_61f29_row1_col0\" class=\"data row1 col0\" >39.8%</td>\n",
       "      <td id=\"T_61f29_row1_col1\" class=\"data row1 col1\" >29.0%</td>\n",
       "      <td id=\"T_61f29_row1_col2\" class=\"data row1 col2\" >0.0%</td>\n",
       "      <td id=\"T_61f29_row1_col3\" class=\"data row1 col3\" >11.2%</td>\n",
       "      <td id=\"T_61f29_row1_col4\" class=\"data row1 col4\" >42.0%</td>\n",
       "      <td id=\"T_61f29_row1_col5\" class=\"data row1 col5\" >61.1%</td>\n",
       "      <td id=\"T_61f29_row1_col6\" class=\"data row1 col6\" >93.4%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_61f29_level1_row2\" class=\"row_heading level1 row2\" >5</th>\n",
       "      <td id=\"T_61f29_row2_col0\" class=\"data row2 col0\" >43.8%</td>\n",
       "      <td id=\"T_61f29_row2_col1\" class=\"data row2 col1\" >27.5%</td>\n",
       "      <td id=\"T_61f29_row2_col2\" class=\"data row2 col2\" >0.0%</td>\n",
       "      <td id=\"T_61f29_row2_col3\" class=\"data row2 col3\" >25.0%</td>\n",
       "      <td id=\"T_61f29_row2_col4\" class=\"data row2 col4\" >47.5%</td>\n",
       "      <td id=\"T_61f29_row2_col5\" class=\"data row2 col5\" >63.8%</td>\n",
       "      <td id=\"T_61f29_row2_col6\" class=\"data row2 col6\" >94.8%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_61f29_level1_row3\" class=\"row_heading level1 row3\" >6</th>\n",
       "      <td id=\"T_61f29_row3_col0\" class=\"data row3 col0\" >52.2%</td>\n",
       "      <td id=\"T_61f29_row3_col1\" class=\"data row3 col1\" >22.2%</td>\n",
       "      <td id=\"T_61f29_row3_col2\" class=\"data row3 col2\" >0.0%</td>\n",
       "      <td id=\"T_61f29_row3_col3\" class=\"data row3 col3\" >35.8%</td>\n",
       "      <td id=\"T_61f29_row3_col4\" class=\"data row3 col4\" >56.7%</td>\n",
       "      <td id=\"T_61f29_row3_col5\" class=\"data row3 col5\" >64.9%</td>\n",
       "      <td id=\"T_61f29_row3_col6\" class=\"data row3 col6\" >95.1%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_61f29_level1_row4\" class=\"row_heading level1 row4\" >7</th>\n",
       "      <td id=\"T_61f29_row4_col0\" class=\"data row4 col0\" >54.6%</td>\n",
       "      <td id=\"T_61f29_row4_col1\" class=\"data row4 col1\" >22.0%</td>\n",
       "      <td id=\"T_61f29_row4_col2\" class=\"data row4 col2\" >5.0%</td>\n",
       "      <td id=\"T_61f29_row4_col3\" class=\"data row4 col3\" >40.0%</td>\n",
       "      <td id=\"T_61f29_row4_col4\" class=\"data row4 col4\" >58.8%</td>\n",
       "      <td id=\"T_61f29_row4_col5\" class=\"data row4 col5\" >66.3%</td>\n",
       "      <td id=\"T_61f29_row4_col6\" class=\"data row4 col6\" >95.1%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_61f29_level0_row5\" class=\"row_heading level0 row5\" rowspan=\"4\">4</th>\n",
       "      <th id=\"T_61f29_level1_row5\" class=\"row_heading level1 row5\" >4</th>\n",
       "      <td id=\"T_61f29_row5_col0\" class=\"data row5 col0\" >26.4%</td>\n",
       "      <td id=\"T_61f29_row5_col1\" class=\"data row5 col1\" >25.1%</td>\n",
       "      <td id=\"T_61f29_row5_col2\" class=\"data row5 col2\" >0.0%</td>\n",
       "      <td id=\"T_61f29_row5_col3\" class=\"data row5 col3\" >0.0%</td>\n",
       "      <td id=\"T_61f29_row5_col4\" class=\"data row5 col4\" >20.0%</td>\n",
       "      <td id=\"T_61f29_row5_col5\" class=\"data row5 col5\" >47.5%</td>\n",
       "      <td id=\"T_61f29_row5_col6\" class=\"data row5 col6\" >92.2%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_61f29_level1_row6\" class=\"row_heading level1 row6\" >5</th>\n",
       "      <td id=\"T_61f29_row6_col0\" class=\"data row6 col0\" >35.1%</td>\n",
       "      <td id=\"T_61f29_row6_col1\" class=\"data row6 col1\" >24.4%</td>\n",
       "      <td id=\"T_61f29_row6_col2\" class=\"data row6 col2\" >0.0%</td>\n",
       "      <td id=\"T_61f29_row6_col3\" class=\"data row6 col3\" >15.0%</td>\n",
       "      <td id=\"T_61f29_row6_col4\" class=\"data row6 col4\" >36.5%</td>\n",
       "      <td id=\"T_61f29_row6_col5\" class=\"data row6 col5\" >53.3%</td>\n",
       "      <td id=\"T_61f29_row6_col6\" class=\"data row6 col6\" >93.2%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_61f29_level1_row7\" class=\"row_heading level1 row7\" >6</th>\n",
       "      <td id=\"T_61f29_row7_col0\" class=\"data row7 col0\" >43.3%</td>\n",
       "      <td id=\"T_61f29_row7_col1\" class=\"data row7 col1\" >22.4%</td>\n",
       "      <td id=\"T_61f29_row7_col2\" class=\"data row7 col2\" >0.0%</td>\n",
       "      <td id=\"T_61f29_row7_col3\" class=\"data row7 col3\" >29.3%</td>\n",
       "      <td id=\"T_61f29_row7_col4\" class=\"data row7 col4\" >45.9%</td>\n",
       "      <td id=\"T_61f29_row7_col5\" class=\"data row7 col5\" >60.0%</td>\n",
       "      <td id=\"T_61f29_row7_col6\" class=\"data row7 col6\" >93.3%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_61f29_level1_row8\" class=\"row_heading level1 row8\" >7</th>\n",
       "      <td id=\"T_61f29_row8_col0\" class=\"data row8 col0\" >47.0%</td>\n",
       "      <td id=\"T_61f29_row8_col1\" class=\"data row8 col1\" >21.8%</td>\n",
       "      <td id=\"T_61f29_row8_col2\" class=\"data row8 col2\" >0.0%</td>\n",
       "      <td id=\"T_61f29_row8_col3\" class=\"data row8 col3\" >30.0%</td>\n",
       "      <td id=\"T_61f29_row8_col4\" class=\"data row8 col4\" >50.0%</td>\n",
       "      <td id=\"T_61f29_row8_col5\" class=\"data row8 col5\" >62.0%</td>\n",
       "      <td id=\"T_61f29_row8_col6\" class=\"data row8 col6\" >92.9%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_61f29_level0_row9\" class=\"row_heading level0 row9\" rowspan=\"3\">5</th>\n",
       "      <th id=\"T_61f29_level1_row9\" class=\"row_heading level1 row9\" >5</th>\n",
       "      <td id=\"T_61f29_row9_col0\" class=\"data row9 col0\" >26.9%</td>\n",
       "      <td id=\"T_61f29_row9_col1\" class=\"data row9 col1\" >23.7%</td>\n",
       "      <td id=\"T_61f29_row9_col2\" class=\"data row9 col2\" >0.0%</td>\n",
       "      <td id=\"T_61f29_row9_col3\" class=\"data row9 col3\" >5.0%</td>\n",
       "      <td id=\"T_61f29_row9_col4\" class=\"data row9 col4\" >23.3%</td>\n",
       "      <td id=\"T_61f29_row9_col5\" class=\"data row9 col5\" >45.0%</td>\n",
       "      <td id=\"T_61f29_row9_col6\" class=\"data row9 col6\" >93.3%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_61f29_level1_row10\" class=\"row_heading level1 row10\" >6</th>\n",
       "      <td id=\"T_61f29_row10_col0\" class=\"data row10 col0\" >38.4%</td>\n",
       "      <td id=\"T_61f29_row10_col1\" class=\"data row10 col1\" >23.1%</td>\n",
       "      <td id=\"T_61f29_row10_col2\" class=\"data row10 col2\" >0.0%</td>\n",
       "      <td id=\"T_61f29_row10_col3\" class=\"data row10 col3\" >20.0%</td>\n",
       "      <td id=\"T_61f29_row10_col4\" class=\"data row10 col4\" >40.0%</td>\n",
       "      <td id=\"T_61f29_row10_col5\" class=\"data row10 col5\" >56.2%</td>\n",
       "      <td id=\"T_61f29_row10_col6\" class=\"data row10 col6\" >100.0%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_61f29_level1_row11\" class=\"row_heading level1 row11\" >7</th>\n",
       "      <td id=\"T_61f29_row11_col0\" class=\"data row11 col0\" >43.1%</td>\n",
       "      <td id=\"T_61f29_row11_col1\" class=\"data row11 col1\" >22.3%</td>\n",
       "      <td id=\"T_61f29_row11_col2\" class=\"data row11 col2\" >0.0%</td>\n",
       "      <td id=\"T_61f29_row11_col3\" class=\"data row11 col3\" >28.0%</td>\n",
       "      <td id=\"T_61f29_row11_col4\" class=\"data row11 col4\" >43.3%</td>\n",
       "      <td id=\"T_61f29_row11_col5\" class=\"data row11 col5\" >60.0%</td>\n",
       "      <td id=\"T_61f29_row11_col6\" class=\"data row11 col6\" >100.0%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_61f29_level0_row12\" class=\"row_heading level0 row12\" rowspan=\"2\">6</th>\n",
       "      <th id=\"T_61f29_level1_row12\" class=\"row_heading level1 row12\" >6</th>\n",
       "      <td id=\"T_61f29_row12_col0\" class=\"data row12 col0\" >38.7%</td>\n",
       "      <td id=\"T_61f29_row12_col1\" class=\"data row12 col1\" >24.2%</td>\n",
       "      <td id=\"T_61f29_row12_col2\" class=\"data row12 col2\" >0.0%</td>\n",
       "      <td id=\"T_61f29_row12_col3\" class=\"data row12 col3\" >20.0%</td>\n",
       "      <td id=\"T_61f29_row12_col4\" class=\"data row12 col4\" >40.0%</td>\n",
       "      <td id=\"T_61f29_row12_col5\" class=\"data row12 col5\" >60.0%</td>\n",
       "      <td id=\"T_61f29_row12_col6\" class=\"data row12 col6\" >100.0%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_61f29_level1_row13\" class=\"row_heading level1 row13\" >7</th>\n",
       "      <td id=\"T_61f29_row13_col0\" class=\"data row13 col0\" >44.3%</td>\n",
       "      <td id=\"T_61f29_row13_col1\" class=\"data row13 col1\" >23.7%</td>\n",
       "      <td id=\"T_61f29_row13_col2\" class=\"data row13 col2\" >0.0%</td>\n",
       "      <td id=\"T_61f29_row13_col3\" class=\"data row13 col3\" >30.0%</td>\n",
       "      <td id=\"T_61f29_row13_col4\" class=\"data row13 col4\" >45.0%</td>\n",
       "      <td id=\"T_61f29_row13_col5\" class=\"data row13 col5\" >60.0%</td>\n",
       "      <td id=\"T_61f29_row13_col6\" class=\"data row13 col6\" >100.0%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_61f29_level0_row14\" class=\"row_heading level0 row14\" >7</th>\n",
       "      <th id=\"T_61f29_level1_row14\" class=\"row_heading level1 row14\" >7</th>\n",
       "      <td id=\"T_61f29_row14_col0\" class=\"data row14 col0\" >40.9%</td>\n",
       "      <td id=\"T_61f29_row14_col1\" class=\"data row14 col1\" >24.7%</td>\n",
       "      <td id=\"T_61f29_row14_col2\" class=\"data row14 col2\" >0.0%</td>\n",
       "      <td id=\"T_61f29_row14_col3\" class=\"data row14 col3\" >20.0%</td>\n",
       "      <td id=\"T_61f29_row14_col4\" class=\"data row14 col4\" >40.0%</td>\n",
       "      <td id=\"T_61f29_row14_col5\" class=\"data row14 col5\" >60.0%</td>\n",
       "      <td id=\"T_61f29_row14_col6\" class=\"data row14 col6\" >100.0%</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7f6f64383ac0>"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Display per-class accuracy statistics.\n",
    "#\n",
    "# Note that there are 10 examples per leaf node.\n",
    "# This makes the numbers round at deep levels.\n",
    "\n",
    "rows = {}\n",
    "for test_depth in exp_dirs:\n",
    "    for train_depth in exp_dirs:\n",
    "        if train_depth < test_depth:\n",
    "            continue\n",
    "        rows[test_depth, train_depth] = pd.DataFrame(\n",
    "            [per_class[train_depth, test_depth]]\n",
    "        ).T.describe().T\n",
    "\n",
    "table = pd.concat(rows)\n",
    "table = table.droplevel(2)\n",
    "table.index = table.index.set_names(['test', 'train'])\n",
    "del table['count']\n",
    "# table.format('{:+.1%}')\n",
    "\n",
    "table.style.format('{:.1%}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "34a7a7d5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style type=\"text/css\">\n",
       "#T_0e4b4_row0_col0, #T_0e4b4_row0_col2, #T_0e4b4_row0_col3, #T_0e4b4_row0_col4, #T_0e4b4_row1_col0, #T_0e4b4_row1_col2, #T_0e4b4_row1_col3, #T_0e4b4_row1_col4, #T_0e4b4_row2_col0, #T_0e4b4_row2_col2, #T_0e4b4_row2_col3, #T_0e4b4_row2_col4, #T_0e4b4_row2_col5, #T_0e4b4_row3_col0, #T_0e4b4_row3_col2, #T_0e4b4_row3_col3, #T_0e4b4_row3_col4, #T_0e4b4_row3_col5, #T_0e4b4_row4_col0, #T_0e4b4_row4_col2, #T_0e4b4_row4_col3, #T_0e4b4_row4_col4, #T_0e4b4_row5_col0, #T_0e4b4_row5_col2, #T_0e4b4_row5_col3, #T_0e4b4_row5_col4, #T_0e4b4_row6_col0, #T_0e4b4_row6_col2, #T_0e4b4_row6_col3, #T_0e4b4_row6_col4, #T_0e4b4_row6_col5, #T_0e4b4_row7_col0, #T_0e4b4_row7_col2, #T_0e4b4_row7_col3, #T_0e4b4_row7_col4, #T_0e4b4_row8_col0, #T_0e4b4_row8_col2, #T_0e4b4_row8_col3, #T_0e4b4_row8_col4, #T_0e4b4_row9_col0, #T_0e4b4_row9_col2, #T_0e4b4_row9_col3 {\n",
       "  color: black;\n",
       "}\n",
       "#T_0e4b4_row0_col5, #T_0e4b4_row0_col6, #T_0e4b4_row1_col5, #T_0e4b4_row1_col6, #T_0e4b4_row2_col6, #T_0e4b4_row3_col6, #T_0e4b4_row4_col5, #T_0e4b4_row4_col6, #T_0e4b4_row5_col5, #T_0e4b4_row5_col6, #T_0e4b4_row6_col6, #T_0e4b4_row7_col5, #T_0e4b4_row7_col6, #T_0e4b4_row8_col5, #T_0e4b4_row8_col6, #T_0e4b4_row9_col4, #T_0e4b4_row9_col5, #T_0e4b4_row9_col6 {\n",
       "  color: green;\n",
       "}\n",
       "</style>\n",
       "<table id=\"T_0e4b4\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th class=\"blank\" >&nbsp;</th>\n",
       "      <th class=\"blank level0\" >&nbsp;</th>\n",
       "      <th id=\"T_0e4b4_level0_col0\" class=\"col_heading level0 col0\" >mean</th>\n",
       "      <th id=\"T_0e4b4_level0_col1\" class=\"col_heading level0 col1\" >std</th>\n",
       "      <th id=\"T_0e4b4_level0_col2\" class=\"col_heading level0 col2\" >min</th>\n",
       "      <th id=\"T_0e4b4_level0_col3\" class=\"col_heading level0 col3\" >50%</th>\n",
       "      <th id=\"T_0e4b4_level0_col4\" class=\"col_heading level0 col4\" >85%</th>\n",
       "      <th id=\"T_0e4b4_level0_col5\" class=\"col_heading level0 col5\" >95%</th>\n",
       "      <th id=\"T_0e4b4_level0_col6\" class=\"col_heading level0 col6\" >max</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th class=\"index_name level0\" >test</th>\n",
       "      <th class=\"index_name level1\" >train</th>\n",
       "      <th class=\"blank col0\" >&nbsp;</th>\n",
       "      <th class=\"blank col1\" >&nbsp;</th>\n",
       "      <th class=\"blank col2\" >&nbsp;</th>\n",
       "      <th class=\"blank col3\" >&nbsp;</th>\n",
       "      <th class=\"blank col4\" >&nbsp;</th>\n",
       "      <th class=\"blank col5\" >&nbsp;</th>\n",
       "      <th class=\"blank col6\" >&nbsp;</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th id=\"T_0e4b4_level0_row0\" class=\"row_heading level0 row0\" rowspan=\"4\">3</th>\n",
       "      <th id=\"T_0e4b4_level1_row0\" class=\"row_heading level1 row0\" >4</th>\n",
       "      <td id=\"T_0e4b4_row0_col0\" class=\"data row0 col0\" >-7.1%</td>\n",
       "      <td id=\"T_0e4b4_row0_col1\" class=\"data row0 col1\" >+10.1%</td>\n",
       "      <td id=\"T_0e4b4_row0_col2\" class=\"data row0 col2\" >-45.0%</td>\n",
       "      <td id=\"T_0e4b4_row0_col3\" class=\"data row0 col3\" >-4.3%</td>\n",
       "      <td id=\"T_0e4b4_row0_col4\" class=\"data row0 col4\" >+0.0%</td>\n",
       "      <td id=\"T_0e4b4_row0_col5\" class=\"data row0 col5\" >+1.7%</td>\n",
       "      <td id=\"T_0e4b4_row0_col6\" class=\"data row0 col6\" >+8.0%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_0e4b4_level1_row1\" class=\"row_heading level1 row1\" >5</th>\n",
       "      <td id=\"T_0e4b4_row1_col0\" class=\"data row1 col0\" >-11.1%</td>\n",
       "      <td id=\"T_0e4b4_row1_col1\" class=\"data row1 col1\" >+12.8%</td>\n",
       "      <td id=\"T_0e4b4_row1_col2\" class=\"data row1 col2\" >-50.0%</td>\n",
       "      <td id=\"T_0e4b4_row1_col3\" class=\"data row1 col3\" >-8.0%</td>\n",
       "      <td id=\"T_0e4b4_row1_col4\" class=\"data row1 col4\" >+0.0%</td>\n",
       "      <td id=\"T_0e4b4_row1_col5\" class=\"data row1 col5\" >+2.2%</td>\n",
       "      <td id=\"T_0e4b4_row1_col6\" class=\"data row1 col6\" >+12.0%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_0e4b4_level1_row2\" class=\"row_heading level1 row2\" >6</th>\n",
       "      <td id=\"T_0e4b4_row2_col0\" class=\"data row2 col0\" >-19.5%</td>\n",
       "      <td id=\"T_0e4b4_row2_col1\" class=\"data row2 col1\" >+16.9%</td>\n",
       "      <td id=\"T_0e4b4_row2_col2\" class=\"data row2 col2\" >-60.0%</td>\n",
       "      <td id=\"T_0e4b4_row2_col3\" class=\"data row2 col3\" >-16.7%</td>\n",
       "      <td id=\"T_0e4b4_row2_col4\" class=\"data row2 col4\" >-4.0%</td>\n",
       "      <td id=\"T_0e4b4_row2_col5\" class=\"data row2 col5\" >-1.7%</td>\n",
       "      <td id=\"T_0e4b4_row2_col6\" class=\"data row2 col6\" >+10.0%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_0e4b4_level1_row3\" class=\"row_heading level1 row3\" >7</th>\n",
       "      <td id=\"T_0e4b4_row3_col0\" class=\"data row3 col0\" >-21.9%</td>\n",
       "      <td id=\"T_0e4b4_row3_col1\" class=\"data row3 col1\" >+17.6%</td>\n",
       "      <td id=\"T_0e4b4_row3_col2\" class=\"data row3 col2\" >-66.7%</td>\n",
       "      <td id=\"T_0e4b4_row3_col3\" class=\"data row3 col3\" >-20.0%</td>\n",
       "      <td id=\"T_0e4b4_row3_col4\" class=\"data row3 col4\" >-4.1%</td>\n",
       "      <td id=\"T_0e4b4_row3_col5\" class=\"data row3 col5\" >-1.8%</td>\n",
       "      <td id=\"T_0e4b4_row3_col6\" class=\"data row3 col6\" >+10.0%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_0e4b4_level0_row4\" class=\"row_heading level0 row4\" rowspan=\"3\">4</th>\n",
       "      <th id=\"T_0e4b4_level1_row4\" class=\"row_heading level1 row4\" >5</th>\n",
       "      <td id=\"T_0e4b4_row4_col0\" class=\"data row4 col0\" >-8.7%</td>\n",
       "      <td id=\"T_0e4b4_row4_col1\" class=\"data row4 col1\" >+12.7%</td>\n",
       "      <td id=\"T_0e4b4_row4_col2\" class=\"data row4 col2\" >-70.0%</td>\n",
       "      <td id=\"T_0e4b4_row4_col3\" class=\"data row4 col3\" >-5.3%</td>\n",
       "      <td id=\"T_0e4b4_row4_col4\" class=\"data row4 col4\" >+0.0%</td>\n",
       "      <td id=\"T_0e4b4_row4_col5\" class=\"data row4 col5\" >+4.1%</td>\n",
       "      <td id=\"T_0e4b4_row4_col6\" class=\"data row4 col6\" >+25.0%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_0e4b4_level1_row5\" class=\"row_heading level1 row5\" >6</th>\n",
       "      <td id=\"T_0e4b4_row5_col0\" class=\"data row5 col0\" >-16.9%</td>\n",
       "      <td id=\"T_0e4b4_row5_col1\" class=\"data row5 col1\" >+17.4%</td>\n",
       "      <td id=\"T_0e4b4_row5_col2\" class=\"data row5 col2\" >-80.0%</td>\n",
       "      <td id=\"T_0e4b4_row5_col3\" class=\"data row5 col3\" >-13.6%</td>\n",
       "      <td id=\"T_0e4b4_row5_col4\" class=\"data row5 col4\" >-0.1%</td>\n",
       "      <td id=\"T_0e4b4_row5_col5\" class=\"data row5 col5\" >+2.1%</td>\n",
       "      <td id=\"T_0e4b4_row5_col6\" class=\"data row5 col6\" >+15.0%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_0e4b4_level1_row6\" class=\"row_heading level1 row6\" >7</th>\n",
       "      <td id=\"T_0e4b4_row6_col0\" class=\"data row6 col0\" >-20.5%</td>\n",
       "      <td id=\"T_0e4b4_row6_col1\" class=\"data row6 col1\" >+18.4%</td>\n",
       "      <td id=\"T_0e4b4_row6_col2\" class=\"data row6 col2\" >-90.0%</td>\n",
       "      <td id=\"T_0e4b4_row6_col3\" class=\"data row6 col3\" >-18.9%</td>\n",
       "      <td id=\"T_0e4b4_row6_col4\" class=\"data row6 col4\" >-2.9%</td>\n",
       "      <td id=\"T_0e4b4_row6_col5\" class=\"data row6 col5\" >+0.0%</td>\n",
       "      <td id=\"T_0e4b4_row6_col6\" class=\"data row6 col6\" >+20.0%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_0e4b4_level0_row7\" class=\"row_heading level0 row7\" rowspan=\"2\">5</th>\n",
       "      <th id=\"T_0e4b4_level1_row7\" class=\"row_heading level1 row7\" >6</th>\n",
       "      <td id=\"T_0e4b4_row7_col0\" class=\"data row7 col0\" >-11.5%</td>\n",
       "      <td id=\"T_0e4b4_row7_col1\" class=\"data row7 col1\" >+14.0%</td>\n",
       "      <td id=\"T_0e4b4_row7_col2\" class=\"data row7 col2\" >-90.0%</td>\n",
       "      <td id=\"T_0e4b4_row7_col3\" class=\"data row7 col3\" >-10.0%</td>\n",
       "      <td id=\"T_0e4b4_row7_col4\" class=\"data row7 col4\" >+0.0%</td>\n",
       "      <td id=\"T_0e4b4_row7_col5\" class=\"data row7 col5\" >+7.1%</td>\n",
       "      <td id=\"T_0e4b4_row7_col6\" class=\"data row7 col6\" >+30.0%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_0e4b4_level1_row8\" class=\"row_heading level1 row8\" >7</th>\n",
       "      <td id=\"T_0e4b4_row8_col0\" class=\"data row8 col0\" >-16.2%</td>\n",
       "      <td id=\"T_0e4b4_row8_col1\" class=\"data row8 col1\" >+16.1%</td>\n",
       "      <td id=\"T_0e4b4_row8_col2\" class=\"data row8 col2\" >-90.0%</td>\n",
       "      <td id=\"T_0e4b4_row8_col3\" class=\"data row8 col3\" >-14.7%</td>\n",
       "      <td id=\"T_0e4b4_row8_col4\" class=\"data row8 col4\" >+0.0%</td>\n",
       "      <td id=\"T_0e4b4_row8_col5\" class=\"data row8 col5\" >+5.0%</td>\n",
       "      <td id=\"T_0e4b4_row8_col6\" class=\"data row8 col6\" >+25.0%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_0e4b4_level0_row9\" class=\"row_heading level0 row9\" >6</th>\n",
       "      <th id=\"T_0e4b4_level1_row9\" class=\"row_heading level1 row9\" >7</th>\n",
       "      <td id=\"T_0e4b4_row9_col0\" class=\"data row9 col0\" >-5.6%</td>\n",
       "      <td id=\"T_0e4b4_row9_col1\" class=\"data row9 col1\" >+13.2%</td>\n",
       "      <td id=\"T_0e4b4_row9_col2\" class=\"data row9 col2\" >-80.0%</td>\n",
       "      <td id=\"T_0e4b4_row9_col3\" class=\"data row9 col3\" >-5.0%</td>\n",
       "      <td id=\"T_0e4b4_row9_col4\" class=\"data row9 col4\" >+10.0%</td>\n",
       "      <td id=\"T_0e4b4_row9_col5\" class=\"data row9 col5\" >+10.0%</td>\n",
       "      <td id=\"T_0e4b4_row9_col6\" class=\"data row9 col6\" >+50.0%</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7f6f11bb8c40>"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Look at difference in per-class accuracy between coarse and fine levels.\n",
    "\n",
    "percentiles = [0.5, 0.85, 0.95]\n",
    "\n",
    "rows = {}\n",
    "for test_depth in exp_dirs:\n",
    "    for train_depth in exp_dirs:\n",
    "        if train_depth <= test_depth:\n",
    "            continue\n",
    "        rows[test_depth, train_depth] = pd.DataFrame(\n",
    "            [per_class[test_depth, test_depth] - per_class[train_depth, test_depth]]\n",
    "        ).T.describe(percentiles).T\n",
    "\n",
    "table = pd.concat(rows)\n",
    "table = table.droplevel(2)\n",
    "table.index = table.index.set_names(['test', 'train'])\n",
    "del table['count']\n",
    "# table.format('{:+.1%}')\n",
    "\n",
    "def highlight(s):\n",
    "    if s.name == 'std':\n",
    "        return None\n",
    "    return ['color: green;' if x > 0 else 'color: black;' for x in s]\n",
    "\n",
    "table.style.apply(highlight).format('{:+.1%}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4efd08ce",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}