figures_and_tables/table-multisurv_evaluation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a id='Top'></a>\n",
    "\n",
    "# MultiSurv results<a class='tocSkip'></a>\n",
    "\n",
    "Evaluation metric results for MultiSurv."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "code_folding": []
   },
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "%load_ext watermark\n",
    "\n",
    "import sys\n",
    "import os\n",
    "\n",
    "import ipywidgets as widgets\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import torch\n",
    "\n",
    "# Make modules in \"src\" dir visible\n",
    "project_dir = os.path.split(os.getcwd())[0]\n",
    "if project_dir not in sys.path:\n",
    "    sys.path.append(os.path.join(project_dir, 'src'))\n",
    "\n",
    "import dataset\n",
    "from model import Model\n",
    "import utils"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "toc": true
   },
   "source": [
    "<h1>Table of Contents<span class=\"tocSkip\"></span></h1>\n",
    "<div class=\"toc\"><ul class=\"toc-item\"><li><span><a href=\"#DataLoader\" data-toc-modified-id=\"DataLoader-1\"><span class=\"toc-item-num\">1&nbsp;&nbsp;</span><code>DataLoader</code></a></span></li><li><span><a href=\"#Model\" data-toc-modified-id=\"Model-2\"><span class=\"toc-item-num\">2&nbsp;&nbsp;</span>Model</a></span><ul class=\"toc-item\"><li><span><a href=\"#Load-weights\" data-toc-modified-id=\"Load-weights-2.1\"><span class=\"toc-item-num\">2.1&nbsp;&nbsp;</span>Load weights</a></span></li></ul></li><li><span><a href=\"#Evaluate\" data-toc-modified-id=\"Evaluate-3\"><span class=\"toc-item-num\">3&nbsp;&nbsp;</span>Evaluate</a></span><ul class=\"toc-item\"><li><span><a href=\"#Write-to-results-table\" data-toc-modified-id=\"Write-to-results-table-3.1\"><span class=\"toc-item-num\">3.1&nbsp;&nbsp;</span>Write to results table</a></span></li><li><span><a href=\"#Check-results-on-all-datasets\" data-toc-modified-id=\"Check-results-on-all-datasets-3.2\"><span class=\"toc-item-num\">3.2&nbsp;&nbsp;</span>Check results on all datasets</a></span></li></ul></li></ul></div>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATA = utils.INPUT_DATA_DIR\n",
    "MODELS = utils.TRAINED_MODEL_DIR\n",
    "\n",
    "device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# `DataLoader`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "073a514ba0d4404a9198ca2fba20c55e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "SelectMultiple(description='Input data', index=(0, 1), options=('clinical', 'mRNA', 'DNAm', 'miRNA', 'CNV', 'w…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "modalities = widgets.SelectMultiple(\n",
    "    options=['clinical', 'mRNA', 'DNAm', 'miRNA', 'CNV', 'wsi'],\n",
    "    index=[0, 1],\n",
    "    rows=6,\n",
    "    description='Input data',\n",
    "    disabled=False\n",
    ")\n",
    "display(modalities)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "# 20-cancer type subset (to compare to Cheerla and Gevaert 2019)\n",
    "\n",
    "cancers = ['BLCA', 'BRCA', 'CESC', 'COAD', 'READ', 'HNSC', 'KICH', 'KIRC', 'KIRP', 'LAML',\n",
    "           'LGG', 'LIHC', 'LUAD', 'LUSC', 'OV', 'PAAD', 'PRAD', 'SKCM', 'STAD', 'THCA', 'UCEC']\n",
    "\n",
    "labels = pd.read_csv('../data/labels.tsv', sep='\\t')\n",
    "print(labels.head(3))\n",
    "\n",
    "# List of patients to exclude: patients with cancers that are not in the subset\n",
    "exclude_cancers = list(labels.loc[~labels['project_id'].isin(cancers), 'submitter_id'])\n",
    "len(exclude_cancers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data modalities:\n",
      "   clinical\n",
      "   mRNA\n",
      "\n",
      "Dataset sizes (# patients):\n",
      "   train: 8880\n",
      "   val: 1109\n",
      "   test: 1092\n",
      "\n",
      "Batch size: 128\n"
     ]
    }
   ],
   "source": [
    "dataloaders = utils.get_dataloaders(\n",
    "    data_location=DATA,\n",
    "    labels_file='../data/labels.tsv',\n",
    "    modalities=modalities.value,\n",
    "    wsi_patch_size=299,\n",
    "    n_wsi_patches=5,\n",
    "#     exclude_patients=exclude_cancers,\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "# prediction_intervals = torch.arange(0., 365 * 21, 365)\n",
    "# prediction_intervals = torch.arange(0., 365 * 11, 365)\n",
    "# prediction_intervals = torch.arange(0., 365 * 10.1, 365 / 2)\n",
    "# prediction_intervals = torch.arange(0., 365 * 6, 365)\n",
    "prediction_intervals = torch.arange(0., 365 * 5.1, 365 / 2)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "labels = [(t, e) for t, e in dataloaders['train'].dataset.label_map.values()]\n",
    "durations = [t for t, _ in labels]\n",
    "events = [e for _, e in labels]\n",
    "\n",
    "prediction_intervals = utils.discretize_time_by_duration_quantiles(durations, events, 20)\n",
    "prediction_intervals = torch.from_numpy(prediction_intervals)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Instantiating MultiSurv model...\n"
     ]
    }
   ],
   "source": [
    "multisurv = Model(dataloaders=dataloaders,\n",
    "#                   output_intervals=prediction_intervals,\n",
    "                  device=device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "clinical_CNV_lr0.005_epoch46_acc0.77.pth\r\n",
      "clinical_DNAm_lr0.005_20_cancers_epoch39_concord0.79.pth\r\n",
      "clinical_DNAm_lr0.005_epoch48_acc0.80.pth\r\n",
      "clinical_lr0.005_epoch49_acc0.78.pth\r\n",
      "clinical_miRNA_lr0.005_epoch45_acc0.78.pth\r\n",
      "clinical_mRNA_DNAm_lr0.0005pre_epoch39_acc0.80.pth\r\n",
      "clinical_mRNA_DNAm_lr0.005a_32_epoch54_acc0.80.pth\r\n",
      "clinical_mRNA_DNAm_lr0.005a_64_epoch40_acc0.81.pth\r\n",
      "clinical_mRNA_DNAm_lr0.005a_epoch32_acc0.80.pth\r\n",
      "clinical_mRNA_DNAm_lr0.005b_32_epoch57_acc0.80.pth\r\n",
      "clinical_mRNA_DNAm_lr0.005b_epoch31_acc0.80.pth\r\n",
      "clinical_mRNA_DNAm_lr0.005c_epoch40_acc0.80.pth\r\n",
      "clinical_mRNA_DNAm_lr0.005d_epoch13_acc0.80.pth\r\n",
      "clinical_mRNA_DNAm_lr0.005d_epoch43_acc0.79.pth\r\n",
      "clinical_mRNA_DNAm_lr0.005e_epoch30_acc0.80.pth\r\n",
      "clinical_mRNA_lr0.001_TEST_LOSS_REDUCTION_epoch51_concord0.80.pth\r\n",
      "clinical_mRNA_lr0.001_TEST_LOSS_REDUCTION_epoch75_concord0.80.pth\r\n",
      "clinical_mRNA_lr0.005_10_1year_intervals_epoch37_acc0.80.pth\r\n",
      "clinical_mRNA_lr0.005_10_half-year_intervals_epoch35_acc0.79.pth\r\n",
      "clinical_mRNA_lr0.005_20_1year_intervals_epoch35_acc0.80.pth\r\n",
      "clinical_mRNA_lr0.005_20_cancers_epoch24_concord0.78.pth\r\n",
      "clinical_mRNA_lr0.005_20_cancers_epoch75_concord0.78.pth\r\n",
      "clinical_mRNA_lr0.005_20_half-year_intervals_epoch47_acc0.80.pth\r\n",
      "clinical_mRNA_lr0.005_5_1year_intervals_epoch39_acc0.80.pth\r\n",
      "clinical_mRNA_lr0.005_discretized_by_duration_quantiles_epoch50_acc0.79.pth\r\n",
      "clinical_mRNA_lr0.005_epoch43_acc0.81.pth\r\n",
      "clinical_mRNA_lr0.005_pycox_LogisticHazard_epoch56_acc0.23.pth\r\n",
      "clinical_mRNA_miRNA_DNAm_CNV_lr0.005_epoch15_acc0.79.pth\r\n",
      "clinical_mRNA_miRNA_DNAm_lr0.005_epoch22_acc0.79.pth\r\n",
      "clinical_wsi5patches299px_lr0.001_epoch58_acc0.78.pth\r\n",
      "clinical_wsi5patches299px_mRNA_miRNA_DNAm_CNV_lr0.001_epoch17_acc0.78.pth\r\n",
      "CNV_lr0.005_epoch20_acc0.64.pth\r\n",
      "DNAm_lr0.005_epoch57_acc0.77.pth\r\n",
      "miRNA_lr0.005_epoch12_acc0.73.pth\r\n",
      "mRNA_DNAm_lr0.005_epoch49_acc0.78.pth\r\n",
      "mRNA_lr0.005_epoch54_acc0.76.pth\r\n",
      "wsi5patches299px_lr0.001_epoch44_concord0.55.pth\r\n"
     ]
    }
   ],
   "source": [
    "!ls -1 /mnt/dataA/multisurv_models/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/mnt/dataA/multisurv_models/wsi5patches299px_lr0.001_epoch44_concord0.55.pth\r\n"
     ]
    }
   ],
   "source": [
    "!ls -1 /mnt/dataA/multisurv_models/wsi*"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "# weights =  'clinical_mRNA_lr0.005_discretized_by_duration_quantiles_epoch50_acc0.79.pth'\n",
    "# weights =  'clinical_mRNA_lr0.005_20_1year_intervals_epoch35_acc0.80.pth'\n",
    "# weights =  'clinical_mRNA_lr0.005_10_1year_intervals_epoch37_acc0.80.pth'\n",
    "# weights =  'clinical_mRNA_lr0.005_20_half-year_intervals_epoch47_acc0.80.pth'\n",
    "# weights =  'clinical_mRNA_lr0.005_5_1year_intervals_epoch39_acc0.80.pth'\n",
    "weights =  'clinical_mRNA_lr0.005_10_half-year_intervals_epoch35_acc0.79.pth'\n",
    "\n",
    "multisurv.load_weights(os.path.join(MODELS, weights))"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "# 20 cancers\n",
    "weights = 'clinical_DNAm_lr0.005_20_cancers_epoch39_concord0.79.pth'\n",
    "multisurv.load_weights(os.path.join(MODELS, weights))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Load model weights:\n",
      "/mnt/dataA/multisurv_models/clinical_mRNA_lr0.005_epoch43_acc0.81.pth\n"
     ]
    }
   ],
   "source": [
    "# Best model\n",
    "multisurv.load_weights(os.path.join(MODELS, 'clinical_mRNA_lr0.005_epoch43_acc0.81.pth'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "clinical\n",
      "mRNA\n"
     ]
    }
   ],
   "source": [
    "for modality in modalities.value:\n",
    "    print(modality)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluate"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "def get_interval_midpoints(intervals):    \n",
    "    return intervals[1:] - np.diff(intervals) / 2"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "%%time\n",
    "\n",
    "# Using custom output intervals\n",
    "\n",
    "performance = utils.Evaluation(model=multisurv, dataset=dataloaders['test'].dataset, device=device)\n",
    "\n",
    "prediction_time_points = get_interval_midpoints(prediction_intervals)\n",
    "performance.run_bootstrap(time_points=prediction_time_points)\n",
    "print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collect patient predictions: 1092/1092\n",
      "\n",
      "Bootstrap\n",
      "---------\n",
      "1000/1000\n",
      "\n",
      "CPU times: user 3min 30s, sys: 6.51 s, total: 3min 36s\n",
      "Wall time: 3min 27s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "# Using MultiSurv's default output intervals\n",
    "performance = utils.Evaluation(model=multisurv, dataset=dataloaders['test'].dataset,\n",
    "                               device=device)\n",
    "performance.run_bootstrap()\n",
    "print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ">> clinical + mRNA <<\n",
      "\n",
      "          Value (95% CI)\n",
      "-----------------------------\n",
      "C-index   0.818 (0.801-0.834)\n",
      "Ctd       0.822 (0.806-0.837)\n",
      "IBS       0.138 (0.126-0.15)\n",
      "INBLL     0.425 (0.393-0.459)\n"
     ]
    }
   ],
   "source": [
    "data_modalities = ' + '.join(modalities.value) if len(modalities.value) > 1 else modalities.value[0]\n",
    "print(f'>> {data_modalities} <<')\n",
    "print()\n",
    "performance.show_results()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ">> clinical + mRNA <<\n",
      "\n",
      "          Value (95% CI)\n",
      "-----------------------------\n",
      "C-index   0.818 (0.801-0.834)\n",
      "Ctd       0.822 (0.806-0.837)\n",
      "IBS       0.138 (0.126-0.15)\n",
      "INBLL     0.425 (0.393-0.459)\n"
     ]
    }
   ],
   "source": [
    "data_modalities = ' + '.join(modalities.value) if len(modalities.value) > 1 else modalities.value[0]\n",
    "print(f'>> {data_modalities} <<')\n",
    "print()\n",
    "performance.show_results(method='empirical')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Write to results table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "results = utils.ResultTable()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>clinical</th>\n",
       "      <th>mRNA</th>\n",
       "      <th>DNAm</th>\n",
       "      <th>miRNA</th>\n",
       "      <th>CNV</th>\n",
       "      <th>wsi</th>\n",
       "      <th>clinical + mRNA</th>\n",
       "      <th>clinical + DNAm</th>\n",
       "      <th>clinical + mRNA + DNAm</th>\n",
       "      <th>clinical + mRNA + DNAm + miRNA + CNV</th>\n",
       "      <th>clinical + mRNA + DNAm + miRNA</th>\n",
       "      <th>clinical + miRNA</th>\n",
       "      <th>clinical + CNV</th>\n",
       "      <th>mRNA + DNAm</th>\n",
       "      <th>clinical + mRNA + DNAm + miRNA + CNV + wsi</th>\n",
       "      <th>clinical + wsi</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Algorithm</th>\n",
       "      <th>Metric</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"4\" valign=\"top\">CPH</th>\n",
       "      <th>C-index</th>\n",
       "      <td>0.796 (0.779-0.813)</td>\n",
       "      <td>0.733 (0.712-0.756)</td>\n",
       "      <td>0.739 (0.719-0.76)</td>\n",
       "      <td>0.676 (0.651-0.7)</td>\n",
       "      <td>0.57 (0.543-0.599)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Ctd</th>\n",
       "      <td>0.796 (0.779-0.813)</td>\n",
       "      <td>0.733 (0.712-0.755)</td>\n",
       "      <td>0.739 (0.719-0.76)</td>\n",
       "      <td>0.676 (0.651-0.7)</td>\n",
       "      <td>0.57 (0.543-0.599)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>IBS</th>\n",
       "      <td>0.143 (0.135-0.154)</td>\n",
       "      <td>0.177 (0.165-0.19)</td>\n",
       "      <td>0.179 (0.165-0.192)</td>\n",
       "      <td>0.186 (0.171-0.202)</td>\n",
       "      <td>0.214 (0.207-0.224)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>INBLL</th>\n",
       "      <td>0.438 (0.414-0.465)</td>\n",
       "      <td>0.528 (0.497-0.558)</td>\n",
       "      <td>0.532 (0.499-0.563)</td>\n",
       "      <td>0.547 (0.511-0.585)</td>\n",
       "      <td>0.617 (0.601-0.64)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"4\" valign=\"top\">RSF</th>\n",
       "      <th>C-index</th>\n",
       "      <td>0.764 (0.744-0.782)</td>\n",
       "      <td>0.718 (0.695-0.741)</td>\n",
       "      <td>0.728 (0.707-0.751)</td>\n",
       "      <td>0.663 (0.638-0.688)</td>\n",
       "      <td>0.604 (0.579-0.63)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Ctd</th>\n",
       "      <td>0.77 (0.751-0.789)</td>\n",
       "      <td>0.719 (0.695-0.741)</td>\n",
       "      <td>0.729 (0.709-0.752)</td>\n",
       "      <td>0.664 (0.639-0.689)</td>\n",
       "      <td>0.604 (0.579-0.63)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>IBS</th>\n",
       "      <td>0.184 (0.179-0.191)</td>\n",
       "      <td>0.191 (0.181-0.2)</td>\n",
       "      <td>0.186 (0.176-0.192)</td>\n",
       "      <td>0.193 (0.183-0.201)</td>\n",
       "      <td>0.217 (0.208-0.225)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>INBLL</th>\n",
       "      <td>0.546 (0.533-0.56)</td>\n",
       "      <td>0.563 (0.537-0.581)</td>\n",
       "      <td>0.55 (0.527-0.564)</td>\n",
       "      <td>0.567 (0.543-0.583)</td>\n",
       "      <td>0.621 (0.602-0.64)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"4\" valign=\"top\">DeepSurv</th>\n",
       "      <th>C-index</th>\n",
       "      <td>0.792 (0.773-0.811)</td>\n",
       "      <td>0.746 (0.722-0.768)</td>\n",
       "      <td>0.76 (0.74-0.78)</td>\n",
       "      <td>0.685 (0.661-0.711)</td>\n",
       "      <td>0.596 (0.571-0.621)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Ctd</th>\n",
       "      <td>0.792 (0.773-0.81)</td>\n",
       "      <td>0.746 (0.722-0.768)</td>\n",
       "      <td>0.759 (0.739-0.78)</td>\n",
       "      <td>0.685 (0.661-0.711)</td>\n",
       "      <td>0.596 (0.571-0.621)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>IBS</th>\n",
       "      <td>0.143 (0.134-0.154)</td>\n",
       "      <td>0.18 (0.159-0.198)</td>\n",
       "      <td>0.177 (0.156-0.203)</td>\n",
       "      <td>0.194 (0.17-0.219)</td>\n",
       "      <td>0.229 (0.215-0.247)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>INBLL</th>\n",
       "      <td>0.436 (0.412-0.464)</td>\n",
       "      <td>0.606 (0.513-0.68)</td>\n",
       "      <td>0.931 (0.684-1.132)</td>\n",
       "      <td>0.61 (0.527-0.689)</td>\n",
       "      <td>0.711 (0.651-0.777)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"4\" valign=\"top\">CoxTime</th>\n",
       "      <th>C-index</th>\n",
       "      <td>0.789 (0.771-0.806)</td>\n",
       "      <td>0.731 (0.707-0.753)</td>\n",
       "      <td>0.747 (0.726-0.767)</td>\n",
       "      <td>0.692 (0.667-0.718)</td>\n",
       "      <td>0.588 (0.562-0.614)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Ctd</th>\n",
       "      <td>0.8 (0.783-0.817)</td>\n",
       "      <td>0.732 (0.708-0.755)</td>\n",
       "      <td>0.747 (0.726-0.767)</td>\n",
       "      <td>0.694 (0.67-0.72)</td>\n",
       "      <td>0.588 (0.562-0.613)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>IBS</th>\n",
       "      <td>0.148 (0.139-0.159)</td>\n",
       "      <td>0.173 (0.159-0.185)</td>\n",
       "      <td>0.172 (0.156-0.191)</td>\n",
       "      <td>0.174 (0.158-0.189)</td>\n",
       "      <td>0.218 (0.205-0.231)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>INBLL</th>\n",
       "      <td>0.448 (0.427-0.475)</td>\n",
       "      <td>0.524 (0.488-0.556)</td>\n",
       "      <td>0.519 (0.475-0.571)</td>\n",
       "      <td>0.521 (0.479-0.563)</td>\n",
       "      <td>0.626 (0.594-0.657)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"4\" valign=\"top\">DeepHit</th>\n",
       "      <th>C-index</th>\n",
       "      <td>0.798 (0.78-0.815)</td>\n",
       "      <td>0.744 (0.72-0.767)</td>\n",
       "      <td>0.733 (0.712-0.754)</td>\n",
       "      <td>0.701 (0.675-0.724)</td>\n",
       "      <td>0.579 (0.553-0.603)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Ctd</th>\n",
       "      <td>0.809 (0.792-0.826)</td>\n",
       "      <td>0.752 (0.728-0.774)</td>\n",
       "      <td>0.737 (0.715-0.758)</td>\n",
       "      <td>0.7 (0.674-0.725)</td>\n",
       "      <td>0.575 (0.549-0.599)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>IBS</th>\n",
       "      <td>0.173 (0.165-0.184)</td>\n",
       "      <td>0.191 (0.18-0.198)</td>\n",
       "      <td>0.194 (0.179-0.208)</td>\n",
       "      <td>0.186 (0.177-0.197)</td>\n",
       "      <td>0.217 (0.212-0.224)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>INBLL</th>\n",
       "      <td>0.517 (0.499-0.542)</td>\n",
       "      <td>0.559 (0.532-0.575)</td>\n",
       "      <td>0.571 (0.529-0.605)</td>\n",
       "      <td>0.548 (0.524-0.575)</td>\n",
       "      <td>0.626 (0.613-0.643)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"4\" valign=\"top\">MTLR</th>\n",
       "      <th>C-index</th>\n",
       "      <td>0.801 (0.782-0.821)</td>\n",
       "      <td>0.742 (0.72-0.764)</td>\n",
       "      <td>0.747 (0.726-0.77)</td>\n",
       "      <td>0.691 (0.664-0.717)</td>\n",
       "      <td>0.592 (0.564-0.62)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Ctd</th>\n",
       "      <td>0.806 (0.787-0.825)</td>\n",
       "      <td>0.745 (0.723-0.766)</td>\n",
       "      <td>0.751 (0.73-0.775)</td>\n",
       "      <td>0.693 (0.666-0.718)</td>\n",
       "      <td>0.594 (0.567-0.623)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>IBS</th>\n",
       "      <td>0.147 (0.134-0.161)</td>\n",
       "      <td>0.178 (0.156-0.196)</td>\n",
       "      <td>0.176 (0.162-0.19)</td>\n",
       "      <td>0.192 (0.169-0.214)</td>\n",
       "      <td>0.218 (0.206-0.232)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>INBLL</th>\n",
       "      <td>0.441 (0.41-0.478)</td>\n",
       "      <td>0.543 (0.472-0.598)</td>\n",
       "      <td>0.542 (0.504-0.584)</td>\n",
       "      <td>0.589 (0.515-0.657)</td>\n",
       "      <td>0.629 (0.597-0.662)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"4\" valign=\"top\">Nnet-survival</th>\n",
       "      <th>C-index</th>\n",
       "      <td>0.805 (0.787-0.822)</td>\n",
       "      <td>0.69 (0.662-0.718)</td>\n",
       "      <td>0.688 (0.665-0.712)</td>\n",
       "      <td>0.583 (0.553-0.613)</td>\n",
       "      <td>0.57 (0.544-0.594)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Ctd</th>\n",
       "      <td>0.81 (0.794-0.828)</td>\n",
       "      <td>0.701 (0.673-0.728)</td>\n",
       "      <td>0.696 (0.673-0.721)</td>\n",
       "      <td>0.588 (0.559-0.618)</td>\n",
       "      <td>0.575 (0.548-0.598)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>IBS</th>\n",
       "      <td>0.132 (0.122-0.147)</td>\n",
       "      <td>0.195 (0.169-0.212)</td>\n",
       "      <td>0.379 (0.356-0.408)</td>\n",
       "      <td>0.29 (0.264-0.317)</td>\n",
       "      <td>0.401 (0.375-0.444)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>INBLL</th>\n",
       "      <td>0.404 (0.38-0.443)</td>\n",
       "      <td>0.724 (0.597-0.832)</td>\n",
       "      <td>1.663 (1.426-1.837)</td>\n",
       "      <td>1.397 (1.079-1.697)</td>\n",
       "      <td>1.996 (1.702-2.169)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"4\" valign=\"top\">MultiSurv</th>\n",
       "      <th>C-index</th>\n",
       "      <td>0.806 (0.79-0.823)</td>\n",
       "      <td>0.755 (0.733-0.777)</td>\n",
       "      <td>0.735 (0.712-0.757)</td>\n",
       "      <td>0.701 (0.675-0.727)</td>\n",
       "      <td>0.618 (0.591-0.643)</td>\n",
       "      <td>0.567 (0.539-0.596)</td>\n",
       "      <td>0.818 (0.802-0.833)</td>\n",
       "      <td>0.805 (0.789-0.823)</td>\n",
       "      <td>0.809 (0.793-0.828)</td>\n",
       "      <td>0.802 (0.784-0.819)</td>\n",
       "      <td>0.797 (0.78-0.814)</td>\n",
       "      <td>0.791 (0.774-0.809)</td>\n",
       "      <td>0.795 (0.778-0.812)</td>\n",
       "      <td>0.757 (0.736-0.778)</td>\n",
       "      <td>0.785 (0.767-0.804)</td>\n",
       "      <td>0.799 (0.78-0.815)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Ctd</th>\n",
       "      <td>0.809 (0.793-0.825)</td>\n",
       "      <td>0.758 (0.735-0.78)</td>\n",
       "      <td>0.736 (0.714-0.759)</td>\n",
       "      <td>0.702 (0.677-0.728)</td>\n",
       "      <td>0.617 (0.591-0.643)</td>\n",
       "      <td>0.569 (0.543-0.597)</td>\n",
       "      <td>0.822 (0.805-0.837)</td>\n",
       "      <td>0.808 (0.791-0.826)</td>\n",
       "      <td>0.81 (0.793-0.829)</td>\n",
       "      <td>0.802 (0.784-0.82)</td>\n",
       "      <td>0.798 (0.781-0.815)</td>\n",
       "      <td>0.792 (0.775-0.81)</td>\n",
       "      <td>0.795 (0.778-0.812)</td>\n",
       "      <td>0.761 (0.739-0.782)</td>\n",
       "      <td>0.787 (0.769-0.806)</td>\n",
       "      <td>0.801 (0.783-0.817)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>IBS</th>\n",
       "      <td>0.143 (0.134-0.155)</td>\n",
       "      <td>0.178 (0.157-0.194)</td>\n",
       "      <td>0.175 (0.156-0.189)</td>\n",
       "      <td>0.179 (0.16-0.199)</td>\n",
       "      <td>0.21 (0.2-0.221)</td>\n",
       "      <td>0.220 (0.206-0.231)</td>\n",
       "      <td>0.138 (0.126-0.15)</td>\n",
       "      <td>0.134 (0.125-0.148)</td>\n",
       "      <td>0.146 (0.135-0.158)</td>\n",
       "      <td>0.149 (0.136-0.162)</td>\n",
       "      <td>0.153 (0.139-0.168)</td>\n",
       "      <td>0.147 (0.136-0.161)</td>\n",
       "      <td>0.14 (0.131-0.152)</td>\n",
       "      <td>0.173 (0.153-0.191)</td>\n",
       "      <td>0.152 (0.14-0.166)</td>\n",
       "      <td>0.148 (0.14-0.158)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>INBLL</th>\n",
       "      <td>0.439 (0.416-0.469)</td>\n",
       "      <td>0.533 (0.475-0.577)</td>\n",
       "      <td>0.525 (0.473-0.564)</td>\n",
       "      <td>0.53 (0.482-0.581)</td>\n",
       "      <td>0.605 (0.582-0.63)</td>\n",
       "      <td>0.624 (0.602-0.639)</td>\n",
       "      <td>0.425 (0.392-0.458)</td>\n",
       "      <td>0.42 (0.392-0.455)</td>\n",
       "      <td>0.444 (0.415-0.476)</td>\n",
       "      <td>0.458 (0.42-0.496)</td>\n",
       "      <td>0.474 (0.431-0.52)</td>\n",
       "      <td>0.449 (0.417-0.492)</td>\n",
       "      <td>0.43 (0.406-0.462)</td>\n",
       "      <td>0.545 (0.478-0.601)</td>\n",
       "      <td>0.463 (0.43-0.504)</td>\n",
       "      <td>0.446 (0.427-0.472)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  clinical                 mRNA  \\\n",
       "Algorithm     Metric                                              \n",
       "CPH           C-index  0.796 (0.779-0.813)  0.733 (0.712-0.756)   \n",
       "              Ctd      0.796 (0.779-0.813)  0.733 (0.712-0.755)   \n",
       "              IBS      0.143 (0.135-0.154)   0.177 (0.165-0.19)   \n",
       "              INBLL    0.438 (0.414-0.465)  0.528 (0.497-0.558)   \n",
       "RSF           C-index  0.764 (0.744-0.782)  0.718 (0.695-0.741)   \n",
       "              Ctd       0.77 (0.751-0.789)  0.719 (0.695-0.741)   \n",
       "              IBS      0.184 (0.179-0.191)    0.191 (0.181-0.2)   \n",
       "              INBLL     0.546 (0.533-0.56)  0.563 (0.537-0.581)   \n",
       "DeepSurv      C-index  0.792 (0.773-0.811)  0.746 (0.722-0.768)   \n",
       "              Ctd       0.792 (0.773-0.81)  0.746 (0.722-0.768)   \n",
       "              IBS      0.143 (0.134-0.154)   0.18 (0.159-0.198)   \n",
       "              INBLL    0.436 (0.412-0.464)   0.606 (0.513-0.68)   \n",
       "CoxTime       C-index  0.789 (0.771-0.806)  0.731 (0.707-0.753)   \n",
       "              Ctd        0.8 (0.783-0.817)  0.732 (0.708-0.755)   \n",
       "              IBS      0.148 (0.139-0.159)  0.173 (0.159-0.185)   \n",
       "              INBLL    0.448 (0.427-0.475)  0.524 (0.488-0.556)   \n",
       "DeepHit       C-index   0.798 (0.78-0.815)   0.744 (0.72-0.767)   \n",
       "              Ctd      0.809 (0.792-0.826)  0.752 (0.728-0.774)   \n",
       "              IBS      0.173 (0.165-0.184)   0.191 (0.18-0.198)   \n",
       "              INBLL    0.517 (0.499-0.542)  0.559 (0.532-0.575)   \n",
       "MTLR          C-index  0.801 (0.782-0.821)   0.742 (0.72-0.764)   \n",
       "              Ctd      0.806 (0.787-0.825)  0.745 (0.723-0.766)   \n",
       "              IBS      0.147 (0.134-0.161)  0.178 (0.156-0.196)   \n",
       "              INBLL     0.441 (0.41-0.478)  0.543 (0.472-0.598)   \n",
       "Nnet-survival C-index  0.805 (0.787-0.822)   0.69 (0.662-0.718)   \n",
       "              Ctd       0.81 (0.794-0.828)  0.701 (0.673-0.728)   \n",
       "              IBS      0.132 (0.122-0.147)  0.195 (0.169-0.212)   \n",
       "              INBLL     0.404 (0.38-0.443)  0.724 (0.597-0.832)   \n",
       "MultiSurv     C-index   0.806 (0.79-0.823)  0.755 (0.733-0.777)   \n",
       "              Ctd      0.809 (0.793-0.825)   0.758 (0.735-0.78)   \n",
       "              IBS      0.143 (0.134-0.155)  0.178 (0.157-0.194)   \n",
       "              INBLL    0.439 (0.416-0.469)  0.533 (0.475-0.577)   \n",
       "\n",
       "                                      DNAm                miRNA  \\\n",
       "Algorithm     Metric                                              \n",
       "CPH           C-index   0.739 (0.719-0.76)    0.676 (0.651-0.7)   \n",
       "              Ctd       0.739 (0.719-0.76)    0.676 (0.651-0.7)   \n",
       "              IBS      0.179 (0.165-0.192)  0.186 (0.171-0.202)   \n",
       "              INBLL    0.532 (0.499-0.563)  0.547 (0.511-0.585)   \n",
       "RSF           C-index  0.728 (0.707-0.751)  0.663 (0.638-0.688)   \n",
       "              Ctd      0.729 (0.709-0.752)  0.664 (0.639-0.689)   \n",
       "              IBS      0.186 (0.176-0.192)  0.193 (0.183-0.201)   \n",
       "              INBLL     0.55 (0.527-0.564)  0.567 (0.543-0.583)   \n",
       "DeepSurv      C-index     0.76 (0.74-0.78)  0.685 (0.661-0.711)   \n",
       "              Ctd       0.759 (0.739-0.78)  0.685 (0.661-0.711)   \n",
       "              IBS      0.177 (0.156-0.203)   0.194 (0.17-0.219)   \n",
       "              INBLL    0.931 (0.684-1.132)   0.61 (0.527-0.689)   \n",
       "CoxTime       C-index  0.747 (0.726-0.767)  0.692 (0.667-0.718)   \n",
       "              Ctd      0.747 (0.726-0.767)    0.694 (0.67-0.72)   \n",
       "              IBS      0.172 (0.156-0.191)  0.174 (0.158-0.189)   \n",
       "              INBLL    0.519 (0.475-0.571)  0.521 (0.479-0.563)   \n",
       "DeepHit       C-index  0.733 (0.712-0.754)  0.701 (0.675-0.724)   \n",
       "              Ctd      0.737 (0.715-0.758)    0.7 (0.674-0.725)   \n",
       "              IBS      0.194 (0.179-0.208)  0.186 (0.177-0.197)   \n",
       "              INBLL    0.571 (0.529-0.605)  0.548 (0.524-0.575)   \n",
       "MTLR          C-index   0.747 (0.726-0.77)  0.691 (0.664-0.717)   \n",
       "              Ctd       0.751 (0.73-0.775)  0.693 (0.666-0.718)   \n",
       "              IBS       0.176 (0.162-0.19)  0.192 (0.169-0.214)   \n",
       "              INBLL    0.542 (0.504-0.584)  0.589 (0.515-0.657)   \n",
       "Nnet-survival C-index  0.688 (0.665-0.712)  0.583 (0.553-0.613)   \n",
       "              Ctd      0.696 (0.673-0.721)  0.588 (0.559-0.618)   \n",
       "              IBS      0.379 (0.356-0.408)   0.29 (0.264-0.317)   \n",
       "              INBLL    1.663 (1.426-1.837)  1.397 (1.079-1.697)   \n",
       "MultiSurv     C-index  0.735 (0.712-0.757)  0.701 (0.675-0.727)   \n",
       "              Ctd      0.736 (0.714-0.759)  0.702 (0.677-0.728)   \n",
       "              IBS      0.175 (0.156-0.189)   0.179 (0.16-0.199)   \n",
       "              INBLL    0.525 (0.473-0.564)   0.53 (0.482-0.581)   \n",
       "\n",
       "                                       CNV                  wsi  \\\n",
       "Algorithm     Metric                                              \n",
       "CPH           C-index   0.57 (0.543-0.599)                  NaN   \n",
       "              Ctd       0.57 (0.543-0.599)                  NaN   \n",
       "              IBS      0.214 (0.207-0.224)                  NaN   \n",
       "              INBLL     0.617 (0.601-0.64)                  NaN   \n",
       "RSF           C-index   0.604 (0.579-0.63)                  NaN   \n",
       "              Ctd       0.604 (0.579-0.63)                  NaN   \n",
       "              IBS      0.217 (0.208-0.225)                  NaN   \n",
       "              INBLL     0.621 (0.602-0.64)                  NaN   \n",
       "DeepSurv      C-index  0.596 (0.571-0.621)                  NaN   \n",
       "              Ctd      0.596 (0.571-0.621)                  NaN   \n",
       "              IBS      0.229 (0.215-0.247)                  NaN   \n",
       "              INBLL    0.711 (0.651-0.777)                  NaN   \n",
       "CoxTime       C-index  0.588 (0.562-0.614)                  NaN   \n",
       "              Ctd      0.588 (0.562-0.613)                  NaN   \n",
       "              IBS      0.218 (0.205-0.231)                  NaN   \n",
       "              INBLL    0.626 (0.594-0.657)                  NaN   \n",
       "DeepHit       C-index  0.579 (0.553-0.603)                  NaN   \n",
       "              Ctd      0.575 (0.549-0.599)                  NaN   \n",
       "              IBS      0.217 (0.212-0.224)                  NaN   \n",
       "              INBLL    0.626 (0.613-0.643)                  NaN   \n",
       "MTLR          C-index   0.592 (0.564-0.62)                  NaN   \n",
       "              Ctd      0.594 (0.567-0.623)                  NaN   \n",
       "              IBS      0.218 (0.206-0.232)                  NaN   \n",
       "              INBLL    0.629 (0.597-0.662)                  NaN   \n",
       "Nnet-survival C-index   0.57 (0.544-0.594)                  NaN   \n",
       "              Ctd      0.575 (0.548-0.598)                  NaN   \n",
       "              IBS      0.401 (0.375-0.444)                  NaN   \n",
       "              INBLL    1.996 (1.702-2.169)                  NaN   \n",
       "MultiSurv     C-index  0.618 (0.591-0.643)  0.567 (0.539-0.596)   \n",
       "              Ctd      0.617 (0.591-0.643)  0.569 (0.543-0.597)   \n",
       "              IBS         0.21 (0.2-0.221)  0.220 (0.206-0.231)   \n",
       "              INBLL     0.605 (0.582-0.63)  0.624 (0.602-0.639)   \n",
       "\n",
       "                           clinical + mRNA      clinical + DNAm  \\\n",
       "Algorithm     Metric                                              \n",
       "CPH           C-index                  NaN                  NaN   \n",
       "              Ctd                      NaN                  NaN   \n",
       "              IBS                      NaN                  NaN   \n",
       "              INBLL                    NaN                  NaN   \n",
       "RSF           C-index                  NaN                  NaN   \n",
       "              Ctd                      NaN                  NaN   \n",
       "              IBS                      NaN                  NaN   \n",
       "              INBLL                    NaN                  NaN   \n",
       "DeepSurv      C-index                  NaN                  NaN   \n",
       "              Ctd                      NaN                  NaN   \n",
       "              IBS                      NaN                  NaN   \n",
       "              INBLL                    NaN                  NaN   \n",
       "CoxTime       C-index                  NaN                  NaN   \n",
       "              Ctd                      NaN                  NaN   \n",
       "              IBS                      NaN                  NaN   \n",
       "              INBLL                    NaN                  NaN   \n",
       "DeepHit       C-index                  NaN                  NaN   \n",
       "              Ctd                      NaN                  NaN   \n",
       "              IBS                      NaN                  NaN   \n",
       "              INBLL                    NaN                  NaN   \n",
       "MTLR          C-index                  NaN                  NaN   \n",
       "              Ctd                      NaN                  NaN   \n",
       "              IBS                      NaN                  NaN   \n",
       "              INBLL                    NaN                  NaN   \n",
       "Nnet-survival C-index                  NaN                  NaN   \n",
       "              Ctd                      NaN                  NaN   \n",
       "              IBS                      NaN                  NaN   \n",
       "              INBLL                    NaN                  NaN   \n",
       "MultiSurv     C-index  0.818 (0.802-0.833)  0.805 (0.789-0.823)   \n",
       "              Ctd      0.822 (0.805-0.837)  0.808 (0.791-0.826)   \n",
       "              IBS       0.138 (0.126-0.15)  0.134 (0.125-0.148)   \n",
       "              INBLL    0.425 (0.392-0.458)   0.42 (0.392-0.455)   \n",
       "\n",
       "                      clinical + mRNA + DNAm  \\\n",
       "Algorithm     Metric                           \n",
       "CPH           C-index                    NaN   \n",
       "              Ctd                        NaN   \n",
       "              IBS                        NaN   \n",
       "              INBLL                      NaN   \n",
       "RSF           C-index                    NaN   \n",
       "              Ctd                        NaN   \n",
       "              IBS                        NaN   \n",
       "              INBLL                      NaN   \n",
       "DeepSurv      C-index                    NaN   \n",
       "              Ctd                        NaN   \n",
       "              IBS                        NaN   \n",
       "              INBLL                      NaN   \n",
       "CoxTime       C-index                    NaN   \n",
       "              Ctd                        NaN   \n",
       "              IBS                        NaN   \n",
       "              INBLL                      NaN   \n",
       "DeepHit       C-index                    NaN   \n",
       "              Ctd                        NaN   \n",
       "              IBS                        NaN   \n",
       "              INBLL                      NaN   \n",
       "MTLR          C-index                    NaN   \n",
       "              Ctd                        NaN   \n",
       "              IBS                        NaN   \n",
       "              INBLL                      NaN   \n",
       "Nnet-survival C-index                    NaN   \n",
       "              Ctd                        NaN   \n",
       "              IBS                        NaN   \n",
       "              INBLL                      NaN   \n",
       "MultiSurv     C-index    0.809 (0.793-0.828)   \n",
       "              Ctd         0.81 (0.793-0.829)   \n",
       "              IBS        0.146 (0.135-0.158)   \n",
       "              INBLL      0.444 (0.415-0.476)   \n",
       "\n",
       "                      clinical + mRNA + DNAm + miRNA + CNV  \\\n",
       "Algorithm     Metric                                         \n",
       "CPH           C-index                                  NaN   \n",
       "              Ctd                                      NaN   \n",
       "              IBS                                      NaN   \n",
       "              INBLL                                    NaN   \n",
       "RSF           C-index                                  NaN   \n",
       "              Ctd                                      NaN   \n",
       "              IBS                                      NaN   \n",
       "              INBLL                                    NaN   \n",
       "DeepSurv      C-index                                  NaN   \n",
       "              Ctd                                      NaN   \n",
       "              IBS                                      NaN   \n",
       "              INBLL                                    NaN   \n",
       "CoxTime       C-index                                  NaN   \n",
       "              Ctd                                      NaN   \n",
       "              IBS                                      NaN   \n",
       "              INBLL                                    NaN   \n",
       "DeepHit       C-index                                  NaN   \n",
       "              Ctd                                      NaN   \n",
       "              IBS                                      NaN   \n",
       "              INBLL                                    NaN   \n",
       "MTLR          C-index                                  NaN   \n",
       "              Ctd                                      NaN   \n",
       "              IBS                                      NaN   \n",
       "              INBLL                                    NaN   \n",
       "Nnet-survival C-index                                  NaN   \n",
       "              Ctd                                      NaN   \n",
       "              IBS                                      NaN   \n",
       "              INBLL                                    NaN   \n",
       "MultiSurv     C-index                  0.802 (0.784-0.819)   \n",
       "              Ctd                       0.802 (0.784-0.82)   \n",
       "              IBS                      0.149 (0.136-0.162)   \n",
       "              INBLL                     0.458 (0.42-0.496)   \n",
       "\n",
       "                      clinical + mRNA + DNAm + miRNA     clinical + miRNA  \\\n",
       "Algorithm     Metric                                                        \n",
       "CPH           C-index                            NaN                  NaN   \n",
       "              Ctd                                NaN                  NaN   \n",
       "              IBS                                NaN                  NaN   \n",
       "              INBLL                              NaN                  NaN   \n",
       "RSF           C-index                            NaN                  NaN   \n",
       "              Ctd                                NaN                  NaN   \n",
       "              IBS                                NaN                  NaN   \n",
       "              INBLL                              NaN                  NaN   \n",
       "DeepSurv      C-index                            NaN                  NaN   \n",
       "              Ctd                                NaN                  NaN   \n",
       "              IBS                                NaN                  NaN   \n",
       "              INBLL                              NaN                  NaN   \n",
       "CoxTime       C-index                            NaN                  NaN   \n",
       "              Ctd                                NaN                  NaN   \n",
       "              IBS                                NaN                  NaN   \n",
       "              INBLL                              NaN                  NaN   \n",
       "DeepHit       C-index                            NaN                  NaN   \n",
       "              Ctd                                NaN                  NaN   \n",
       "              IBS                                NaN                  NaN   \n",
       "              INBLL                              NaN                  NaN   \n",
       "MTLR          C-index                            NaN                  NaN   \n",
       "              Ctd                                NaN                  NaN   \n",
       "              IBS                                NaN                  NaN   \n",
       "              INBLL                              NaN                  NaN   \n",
       "Nnet-survival C-index                            NaN                  NaN   \n",
       "              Ctd                                NaN                  NaN   \n",
       "              IBS                                NaN                  NaN   \n",
       "              INBLL                              NaN                  NaN   \n",
       "MultiSurv     C-index             0.797 (0.78-0.814)  0.791 (0.774-0.809)   \n",
       "              Ctd                0.798 (0.781-0.815)   0.792 (0.775-0.81)   \n",
       "              IBS                0.153 (0.139-0.168)  0.147 (0.136-0.161)   \n",
       "              INBLL               0.474 (0.431-0.52)  0.449 (0.417-0.492)   \n",
       "\n",
       "                            clinical + CNV          mRNA + DNAm  \\\n",
       "Algorithm     Metric                                              \n",
       "CPH           C-index                  NaN                  NaN   \n",
       "              Ctd                      NaN                  NaN   \n",
       "              IBS                      NaN                  NaN   \n",
       "              INBLL                    NaN                  NaN   \n",
       "RSF           C-index                  NaN                  NaN   \n",
       "              Ctd                      NaN                  NaN   \n",
       "              IBS                      NaN                  NaN   \n",
       "              INBLL                    NaN                  NaN   \n",
       "DeepSurv      C-index                  NaN                  NaN   \n",
       "              Ctd                      NaN                  NaN   \n",
       "              IBS                      NaN                  NaN   \n",
       "              INBLL                    NaN                  NaN   \n",
       "CoxTime       C-index                  NaN                  NaN   \n",
       "              Ctd                      NaN                  NaN   \n",
       "              IBS                      NaN                  NaN   \n",
       "              INBLL                    NaN                  NaN   \n",
       "DeepHit       C-index                  NaN                  NaN   \n",
       "              Ctd                      NaN                  NaN   \n",
       "              IBS                      NaN                  NaN   \n",
       "              INBLL                    NaN                  NaN   \n",
       "MTLR          C-index                  NaN                  NaN   \n",
       "              Ctd                      NaN                  NaN   \n",
       "              IBS                      NaN                  NaN   \n",
       "              INBLL                    NaN                  NaN   \n",
       "Nnet-survival C-index                  NaN                  NaN   \n",
       "              Ctd                      NaN                  NaN   \n",
       "              IBS                      NaN                  NaN   \n",
       "              INBLL                    NaN                  NaN   \n",
       "MultiSurv     C-index  0.795 (0.778-0.812)  0.757 (0.736-0.778)   \n",
       "              Ctd      0.795 (0.778-0.812)  0.761 (0.739-0.782)   \n",
       "              IBS       0.14 (0.131-0.152)  0.173 (0.153-0.191)   \n",
       "              INBLL     0.43 (0.406-0.462)  0.545 (0.478-0.601)   \n",
       "\n",
       "                      clinical + mRNA + DNAm + miRNA + CNV + wsi  \\\n",
       "Algorithm     Metric                                               \n",
       "CPH           C-index                                        NaN   \n",
       "              Ctd                                            NaN   \n",
       "              IBS                                            NaN   \n",
       "              INBLL                                          NaN   \n",
       "RSF           C-index                                        NaN   \n",
       "              Ctd                                            NaN   \n",
       "              IBS                                            NaN   \n",
       "              INBLL                                          NaN   \n",
       "DeepSurv      C-index                                        NaN   \n",
       "              Ctd                                            NaN   \n",
       "              IBS                                            NaN   \n",
       "              INBLL                                          NaN   \n",
       "CoxTime       C-index                                        NaN   \n",
       "              Ctd                                            NaN   \n",
       "              IBS                                            NaN   \n",
       "              INBLL                                          NaN   \n",
       "DeepHit       C-index                                        NaN   \n",
       "              Ctd                                            NaN   \n",
       "              IBS                                            NaN   \n",
       "              INBLL                                          NaN   \n",
       "MTLR          C-index                                        NaN   \n",
       "              Ctd                                            NaN   \n",
       "              IBS                                            NaN   \n",
       "              INBLL                                          NaN   \n",
       "Nnet-survival C-index                                        NaN   \n",
       "              Ctd                                            NaN   \n",
       "              IBS                                            NaN   \n",
       "              INBLL                                          NaN   \n",
       "MultiSurv     C-index                        0.785 (0.767-0.804)   \n",
       "              Ctd                            0.787 (0.769-0.806)   \n",
       "              IBS                             0.152 (0.14-0.166)   \n",
       "              INBLL                           0.463 (0.43-0.504)   \n",
       "\n",
       "                            clinical + wsi  \n",
       "Algorithm     Metric                        \n",
       "CPH           C-index                  NaN  \n",
       "              Ctd                      NaN  \n",
       "              IBS                      NaN  \n",
       "              INBLL                    NaN  \n",
       "RSF           C-index                  NaN  \n",
       "              Ctd                      NaN  \n",
       "              IBS                      NaN  \n",
       "              INBLL                    NaN  \n",
       "DeepSurv      C-index                  NaN  \n",
       "              Ctd                      NaN  \n",
       "              IBS                      NaN  \n",
       "              INBLL                    NaN  \n",
       "CoxTime       C-index                  NaN  \n",
       "              Ctd                      NaN  \n",
       "              IBS                      NaN  \n",
       "              INBLL                    NaN  \n",
       "DeepHit       C-index                  NaN  \n",
       "              Ctd                      NaN  \n",
       "              IBS                      NaN  \n",
       "              INBLL                    NaN  \n",
       "MTLR          C-index                  NaN  \n",
       "              Ctd                      NaN  \n",
       "              IBS                      NaN  \n",
       "              INBLL                    NaN  \n",
       "Nnet-survival C-index                  NaN  \n",
       "              Ctd                      NaN  \n",
       "              IBS                      NaN  \n",
       "              INBLL                    NaN  \n",
       "MultiSurv     C-index   0.799 (0.78-0.815)  \n",
       "              Ctd      0.801 (0.783-0.817)  \n",
       "              IBS       0.148 (0.14-0.158)  \n",
       "              INBLL    0.446 (0.427-0.472)  "
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_modalities = ' + '.join(modalities.value) if len(modalities.value) > 1 else modalities.value[0]\n",
    "\n",
    "results.write_result_dict(result_dict=performance.format_results(),\n",
    "                          algorithm='MultiSurv',\n",
    "                          data_modality=data_modalities)\n",
    "results.table"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Check results on all datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "~~~~~~~~~~~~~~~~~~~~~~~\n",
      "     RESULT CHECK\n",
      "~~~~~~~~~~~~~~~~~~~~~~~\n",
      "~ train ~\n",
      "Collect patient predictions: 8880/8880\n",
      "\n",
      "C-index   0.843\n",
      "Ctd       0.85\n",
      "IBS       0.112\n",
      "INBLL     0.349\n",
      "\n",
      "~ val ~\n",
      "Collect patient predictions: 1109/1109\n",
      "\n",
      "C-index   0.805\n",
      "Ctd       0.808\n",
      "IBS       0.137\n",
      "INBLL     0.423\n",
      "\n",
      "~ test ~\n",
      "Collect patient predictions: 1092/1092\n",
      "\n",
      "C-index   0.818\n",
      "Ctd       0.822\n",
      "IBS       0.138\n",
      "INBLL     0.425\n",
      "\n",
      "CPU times: user 5min 36s, sys: 1min 18s, total: 6min 55s\n",
      "Wall time: 13min 54s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "print('~' * 23)\n",
    "print('     RESULT CHECK')\n",
    "print('~' * 23)\n",
    "check_results = {'train': None, 'val': None, 'test': None}\n",
    "\n",
    "for group in check_results.keys():\n",
    "    print(f'~ {group} ~')\n",
    "    performance = utils.Evaluation(model=multisurv, dataset=dataloaders[group].dataset, device=device)\n",
    "    performance.compute_metrics()\n",
    "    performance.show_results()\n",
    "    print()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Watermark<a class='tocSkip'></a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pandas     1.0.1\n",
      "torch      1.4.0\n",
      "numpy      1.18.1\n",
      "ipywidgets 7.5.1\n",
      "\n",
      "CPython 3.6.7\n",
      "IPython 7.11.1\n",
      "\n",
      "last updated: Tue Jul 28 2020\n"
     ]
    }
   ],
   "source": [
    "%watermark --iversions\n",
    "%watermark -v\n",
    "print()\n",
    "%watermark -u -n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "[Top of the page](#Top)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {
    "height": "197px",
    "width": "372px"
   },
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": true,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "180px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}