training-stats-analysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "foldername = \"results/sceptr/eval/trained-sceptr-caneval-6\"\n",
    "foldername = \"trained\"\n",
    "rolling_window = 20\n",
    "endofepoch = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils.retrieve_stats import *\n",
    "\n",
    "results = getstats(foldername, rolling_window=rolling_window, endofepoch=endofepoch)\n",
    "plt.rcParams['figure.figsize'] = [20, 10]\n",
    "plt.rcParams[\"figure.autolayout\"] = True\n",
    "\n",
    "stats, epochwisestats, aucstats, epochwiseauc = list(results.values())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, axs = plt.subplots(ncols = 2, nrows = 2)\n",
    "axs[0, 0].plot(stats[\"train-loss.csv\"])\n",
    "axs[0, 0].set_title(\"Training Loss\")\n",
    "\n",
    "axs[1, 0].plot(stats[\"test-loss.csv\"])\n",
    "axs[1, 0].set_title(\"Testing Loss\")\n",
    "\n",
    "axs[0, 1].plot(stats[\"train-acc.csv\"])\n",
    "axs[0, 1].set_title(\"Training Accuracy\")\n",
    "\n",
    "axs[1, 1].plot(stats[\"test-acc.csv\"])\n",
    "axs[1, 1].set_title(\"Testing Accuracy\")\n",
    "\n",
    "fig.suptitle(f\"Batchwise Statistics, Rolling Average Window: {rolling_window}\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, axs = plt.subplots(ncols = 2, nrows = 2)\n",
    "axs[0, 0].plot(epochwisestats[\"train-loss.csv\"])\n",
    "axs[0, 0].set_title(\"Training Loss\")\n",
    "\n",
    "axs[1, 0].plot(epochwisestats[\"test-loss.csv\"])\n",
    "axs[1, 0].set_title(\"Testing Loss\")\n",
    "\n",
    "axs[0, 1].plot(epochwisestats[\"train-acc.csv\"])\n",
    "axs[0, 1].set_title(\"Training Accuracy\")\n",
    "\n",
    "axs[1, 1].plot(epochwisestats[\"test-acc.csv\"])\n",
    "axs[1, 1].set_title(\"Testing Accuracy\")\n",
    "\n",
    "fig.suptitle(f\"Epochwise Statistics\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for key, val in epochwisestats.items():\n",
    "    plt.plot(val, label = \" \".join(key.replace(\".csv\", \"\").split(\"-\")).title(), marker = \"x\")\n",
    "plt.title(\"Epochwise Statistics\")\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, axs = plt.subplots(ncols = 2)\n",
    "\n",
    "for key, val in epochwisestats.items():\n",
    "    axs[0 if \"train\" in key else 1].plot(val, label = \" \".join(key.replace(\".csv\", \"\").split(\"-\")).title(), marker = \"x\")\n",
    "axs[0].legend()\n",
    "axs[0].set_title(\"Train\")\n",
    "axs[1].legend()\n",
    "axs[1].set_title(\"Test\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, axs = plt.subplots(ncols = 2)\n",
    "\n",
    "for key, val in epochwisestats.items():\n",
    "    axs[0 if \"loss\" in key else 1].plot(val, label = \" \".join(key.replace(\".csv\", \"\").split(\"-\")).title(), marker = \"x\")\n",
    "axs[0].legend()\n",
    "axs[1].legend()\n",
    "axs[0].set_title(\"Loss\")\n",
    "axs[1].set_title(\"Accuracy\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for key, val in aucstats.items():\n",
    "    t = key.replace(\".csv\", \"\").split(\"-\")[0]\n",
    "    plt.plot(val, label = f\"{t} AUC\", marker = \"x\")\n",
    "\n",
    "plt.title(\"AUC\")\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, axs = plt.subplots(ncols = 2)\n",
    "\n",
    "axs[0].plot([0, 1], [0, 1], \"k--\", label = \"Random Predictions Curve\")\n",
    "axs[1].plot([0, 1], [0, 1], \"k--\", label = \"Random Predictions Curve\")\n",
    "\n",
    "try:\n",
    "    for epoch in range(len(list(aucstats.values())[0])):\n",
    "        for key in epochwiseauc.keys():\n",
    "            fpr, tpr, thresholds = epochwiseauc[key][epoch]\n",
    "            axs[0 if \"train\" in key else 1].plot(fpr, tpr, label = f\"Epoch {epoch} ROC curve (area = {aucstats[key][epoch]:.5f})\")\n",
    "            axs[0 if \"train\" in key else 1].set_xlim([-0.01, 1])\n",
    "            axs[0 if \"train\" in key else 1].set_ylim([0, 1.01])\n",
    "            axs[0 if \"train\" in key else 1].set_xlabel(\"False Positive Rate\")\n",
    "            axs[0 if \"train\" in key else 1].set_ylabel(\"True Positive Rate\")\n",
    "except KeyError:\n",
    "    pass\n",
    "\n",
    "axs[0].legend()\n",
    "axs[1].legend()\n",
    "axs[0].set_title(\"Training AUC\")\n",
    "axs[1].set_title(\"Testing AUC\")\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tcr-cancer",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}