density_evaluate_markers.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.metrics import balanced_accuracy_score\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.metrics import confusion_matrix\n",
    "import seaborn as sns\n",
    "from artifact_detector_model import MARKER_NAMES"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_markers = pd.read_csv(\"predicted_all_embed.csv\")[[\"image_path\"] + MARKER_NAMES]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_main = pd.read_csv(\n",
    "    \"/vol/biomedic3/data/EMBED/tables/mammo-net-csv/embed-non-negative.csv\"\n",
    ")\n",
    "output_dir = \"output/density-balanced/version_1\"\n",
    "print(len(df_main))\n",
    "df_main[\"image_id\"] = df_main[\"image_path\"].apply(\n",
    "    lambda img_path: img_path.split(\"/\")[-1]\n",
    ")\n",
    "df_markers[\"image_id\"] = df_markers[\"image_path\"].apply(\n",
    "    lambda img_path: img_path.split(\"/\")[-1]\n",
    ")\n",
    "df_main.drop(columns=\"image_path\", inplace=True)\n",
    "df_full = df_main.merge(df_markers, how=\"inner\")\n",
    "df_full = df_full.loc[df_full[\"compression\"] == 0]\n",
    "print(len(df_full))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from ast import literal_eval\n",
    "\n",
    "predictions = pd.read_csv(output_dir + \"/predictions.csv\")\n",
    "predictions[\"probability\"] = predictions[\"probability\"].apply(\n",
    "    lambda x: \" \".join(x.split()).replace(\" \", \",\")\n",
    ")\n",
    "predictions[\"probability\"] = predictions[\"probability\"].apply(\n",
    "    lambda x: np.asarray(literal_eval(x))\n",
    ")\n",
    "predictions[\"image_id\"] = predictions[\"image_id\"].apply(\n",
    "    lambda img_path: img_path.split(\"/\")[-1]\n",
    ")\n",
    "img_ids = predictions[\"image_id\"].values\n",
    "preds = np.stack(predictions[\"probability\"].values)\n",
    "targets = predictions[\"label\"].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "circle_image_id = df_full.loc[\n",
    "    (df_full[\"image_id\"].isin(img_ids)) & (df_full[\"circle marker\"] == 1), \"image_id\"\n",
    "]\n",
    "circle_idx = np.where([img_id in circle_image_id.values for img_id in img_ids])[0]\n",
    "triangle_image_id = df_full.loc[\n",
    "    (df_full[\"image_id\"].isin(img_ids)) & (df_full[\"triangle marker\"] == 1), \"image_id\"\n",
    "]\n",
    "triangle_idx = np.where([img_id in triangle_image_id.values for img_id in img_ids])[0]\n",
    "pacemaker_image_id = df_full.loc[\n",
    "    (df_full[\"image_id\"].isin(img_ids)) & (df_full[\"devices\"] == 1), \"image_id\"\n",
    "]\n",
    "pacemaker_idx = np.where([img_id in pacemaker_image_id.values for img_id in img_ids])[0]\n",
    "breast_implant_image_id = df_full.loc[\n",
    "    (df_full[\"image_id\"].isin(img_ids)) & (df_full[\"breast implant\"] == 1), \"image_id\"\n",
    "]\n",
    "breast_implant_idx = np.where(\n",
    "    [img_id in breast_implant_image_id.values for img_id in img_ids]\n",
    ")[0]\n",
    "compression_image_id = df_full.loc[\n",
    "    (df_full[\"image_id\"].isin(img_ids)) & (df_full[\"compression\"] == 1), \"image_id\"\n",
    "]\n",
    "compression_idx = np.where(\n",
    "    [img_id in compression_image_id.values for img_id in img_ids]\n",
    ")[0]\n",
    "normal_image_id = df_full.loc[\n",
    "    (df_full[\"image_id\"].isin(img_ids))\n",
    "    & (df_full[\"compression\"] == 0)\n",
    "    & (df_full[\"devices\"] == 0)\n",
    "    & (df_full[\"circle marker\"] == 0)\n",
    "    & (df_full[\"triangle marker\"] == 0)\n",
    "    & (df_full[\"breast implant\"] == 0),\n",
    "    \"image_id\",\n",
    "]\n",
    "normal_idx = np.where([img_id in normal_image_id.values for img_id in img_ids])[0]\n",
    "circle_idx.shape, triangle_idx.shape, pacemaker_idx.shape, breast_implant_idx.shape, compression_idx.shape, normal_idx.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\n",
    "    f\"All - balanced accuracy: {balanced_accuracy_score(targets, np.argmax(preds, 1)):.3f}\"\n",
    ")\n",
    "print(\n",
    "    f\"Normal images: {balanced_accuracy_score(targets[normal_idx], np.argmax(preds, 1)[normal_idx]):.3f}\"\n",
    ")\n",
    "print(\n",
    "    f\"Images with circle: {balanced_accuracy_score(targets[circle_idx], np.argmax(preds, 1)[circle_idx]):.3f}\"\n",
    ")\n",
    "print(\n",
    "    f\"Images with triangle: {balanced_accuracy_score(targets[triangle_idx], np.argmax(preds, 1)[triangle_idx]):.3f}\"\n",
    ")\n",
    "print(\n",
    "    f\"Images with implant: {balanced_accuracy_score(targets[breast_implant_idx], np.argmax(preds, 1)[breast_implant_idx]):.3f}\"\n",
    ")\n",
    "print(\n",
    "    f\"Pacemaker: {balanced_accuracy_score(targets[pacemaker_idx], np.argmax(preds, 1)[pacemaker_idx]):.3f}\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "plt.rcParams[\"font.family\"] = \"Serif\"\n",
    "f, ax = plt.subplots(2, 3, figsize=(12, 8), facecolor=\"none\")\n",
    "f.subplots_adjust(hspace=0.4, wspace=0.3)\n",
    "ax = ax.ravel()\n",
    "results = {\n",
    "    'all': [np.arange(targets.shape[0]), \"All images\"],\n",
    "    'no markers': [normal_idx, \"No markers\"],\n",
    "    'circle': [circle_idx, \"Circle markers\"],\n",
    "    'triangles': [triangle_idx, \"Triangle markers\"],\n",
    "    'implants': [breast_implant_idx, \"Breast implants\"],\n",
    "    'devices': [pacemaker_idx, \"Devices\"]\n",
    "}\n",
    "for i, (select_idx, title) in enumerate(results.values()):\n",
    "    cf_matrix = confusion_matrix(targets[select_idx], np.argmax(preds[select_idx], 1))\n",
    "    group_counts = [f\"{value:0.0f}\" for value in cf_matrix.flatten()]\n",
    "    group_percentages = [\n",
    "        f\"{value:.0%}\"\n",
    "        for value in (cf_matrix / np.sum(cf_matrix, 1, keepdims=True)).flatten()\n",
    "    ]\n",
    "    labels = [f\"{v3}\\n(N={v2})\" for v2, v3 in zip(group_counts, group_percentages)]\n",
    "    labels = [f\"{v3}\" for v2, v3 in zip(group_counts, group_percentages)]\n",
    "    labels = np.asarray(labels).reshape(4, 4)\n",
    "    sns.heatmap(\n",
    "        (cf_matrix / np.sum(cf_matrix, 1, keepdims=True)),\n",
    "        annot=labels,\n",
    "        fmt=\"\",\n",
    "        cmap=\"Blues\",\n",
    "        ax=ax[i],\n",
    "        vmin=0,\n",
    "        vmax=1,\n",
    "        cbar=False,\n",
    "        annot_kws={'fontsize': 17}\n",
    "    )\n",
    "    ax[i].set_title(' '.join([r'$\\bf{' + t +'}$' for t in title.split(' ')]), fontsize=20)\n",
    "[ax[i].set_xlabel('') for i in range(3)]\n",
    "[ax[i].set_ylabel('') for i in [1, 2, 4, 5]]\n",
    "[ax[i].set_ylabel('True label', fontsize=20) for i in [0,3]]\n",
    "[ax[i].set_xlabel('Predicted label', fontsize=20) for i in [3,4,5]]\n",
    "\n",
    "plt.savefig(\"output/confusion_density.pdf\", bbox_inches=\"tight\");"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}