manually_inspect_masks_widget.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This notebook is used to inspect embryos after mask was created.  \n",
    "For each embryo the widget displays the max projection of the fish, dapi, and gfp channels + mask.  \n",
    "The user selects which embryos are valid and if the mask has to be fixed.  \n",
    "In the second part of the notebook, the the embryos.csv \"status\" column will update accordingly, and csv is saved.  \n",
    "\n",
    "Please GIT COMMIT csv after changes.  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from glob import glob\n",
    "import os\n",
    "from skimage import io\n",
    "\n",
    "import ipywidgets as widgets\n",
    "from functools import partial\n",
    "\n",
    "import time\n",
    "\n",
    "from IPython.display import clear_output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "csv_path = '/PATH0'\n",
    "preview_path = 'PATH1'\n",
    "save_txt_dir = 'PATH2'\n",
    "preview_png_dir = preview_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "timestr = time.strftime(\"%Y%m%d-%H%M%S\")\n",
    "save_results_txt_path = os.path.join(save_txt_dir, f'embryos_status_updates_{timestr}.txt')\n",
    "fix_masks_txt_path = os.path.join(save_txt_dir, f'embryos_masks_to_fix_{timestr}.txt')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Read CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "csv_file = pd.read_csv(csv_path)\n",
    "csv_file = csv_file.reset_index(drop=True)\n",
    "\n",
    "# Find all the rows/embryos to predict stage:\n",
    "#df_embryos_to_predict = csv_file[(csv_file[\"#nucs_predicted\"]==-1) & ((csv_file[\"status\"]==1) | (csv_file[\"status\"]==-1)) & (csv_file[\"#channels\"]>3)]\n",
    "\n",
    "df_to_view = csv_file[(csv_file[\"status\"]==0) & (~csv_file[\"cropped_image_file\"].isna()) & (csv_file[\"#channels\"]>3)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Check in all files have previews:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_preview_paths = glob(os.path.join(preview_path,'*.png'))\n",
    "all_preview_names = [os.path.basename(f)[:-4] for f in all_preview_paths]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_names = [df_to_view.at[i,\"cropped_image_file\"][:-4] for i in df_to_view.index]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "missing files in preview dir:\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "set()"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print('missing files in preview dir:')\n",
    "set(df_names) - set(all_preview_names)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Create Preview PNG:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for n in df_names:\n",
    "#     path_png = os.path.join(preview_png_dir,f'{n[:-4]}.png')\n",
    "#     if not os.path.exists(path_png):\n",
    "#         im = io.imread(os.path.join(preview_path,n))\n",
    "#         io.imsave(path_png, im)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Show Preview:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "def on_button_clicked(curr_img, texts, buttons, buttons1, b):\n",
    "    \n",
    "    last_beg = curr_img-images_per_page\n",
    "        \n",
    "    with open(save_results_txt_path,\"a\") as f:\n",
    "        f.write(\"\\n\")\n",
    "        f.write(\"\\n\".join([f'{texts[i].value},{buttons[i].value}' for i in range(len(buttons))]))\n",
    "    \n",
    "    with open(fix_masks_txt_path,\"a\") as f:\n",
    "        f.write(\"\\n\")\n",
    "        f.write(\"\\n\".join([f'{texts[i].value},{buttons1[i].value}' for i in range(len(buttons1))]))\n",
    "        \n",
    "    clear_output()\n",
    "    \n",
    "    display(select_images_widget(curr_img))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "def select_images_widget(curr_img_num):\n",
    "    COLS = 5\n",
    "    ROWS = int(images_per_page/COLS)\n",
    "    \n",
    "    IMG_WIDTH = 200\n",
    "    IMG_HEIGHT = 100\n",
    "    \n",
    "    texts = []\n",
    "    images = []\n",
    "    buttons = []\n",
    "    buttons1 = []\n",
    "    \n",
    "    rows = []\n",
    "    \n",
    "    for row in range(ROWS):\n",
    "        cols = []\n",
    "        for col in range(COLS):\n",
    "\n",
    "            index = (row * COLS + col)\n",
    "            \n",
    "            im_name = df_names[index+curr_img_num]\n",
    "            \n",
    "            file = open(os.path.join(preview_png_dir,f'{im_name}.png'), \"rb\")\n",
    "            image = file.read()\n",
    "\n",
    "            texts.append(widgets.Text(value=im_name))\n",
    "            \n",
    "            images.append(widgets.Image(\n",
    "                value=image, width=IMG_WIDTH, height=IMG_HEIGHT)\n",
    "            )\n",
    "\n",
    "            buttons.append(widgets.ToggleButton(value=True,\n",
    "                                          description='Use embryo',\n",
    "                                          disabled=False,\n",
    "                                          #button_style='', # 'success', 'info', 'warning', 'danger' or ''\n",
    "                                          tooltip='Description',\n",
    "                                          icon='check'))\n",
    "            \n",
    "            buttons1.append(widgets.ToggleButton(value=False,\n",
    "                              description='Fix mask?',\n",
    "                              disabled=False,\n",
    "                              #button_style='', # 'success', 'info', 'warning', 'danger' or ''\n",
    "                              tooltip='Description',\n",
    "                              icon='check'))\n",
    "            \n",
    "            box_layout = widgets.Layout(display='flex',\n",
    "                            flex_flow='column',\n",
    "                            width='50%')\n",
    "            \n",
    "            # Create a vertical layout box, image above the button\n",
    "            box = widgets.VBox([texts[index], images[index], buttons[index], buttons1[index]], layout=box_layout)\n",
    "            cols.append(box)\n",
    "\n",
    "        # Create a horizontal layout box, grouping all the columns together\n",
    "        rows.append(widgets.HBox(cols))\n",
    "        \n",
    "    button = widgets.Button(description=\"Next..\")\n",
    "    button.on_click(partial(on_button_clicked, curr_img_num+images_per_page, texts, buttons, buttons1))\n",
    "    rows.append(button)\n",
    "    \n",
    "    # Create a vertical layout box, grouping all the rows together\n",
    "    results = widgets.VBox(rows)\n",
    "    return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ff6481bcb49b466e90356faf3063be0a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HBox(children=(VBox(children=(Text(value='RNAi_dpy27_3840_cropped_0'), Image(value=b'\\x89PNG\\r\\…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "images_per_page = 20\n",
    "select_images_widget(0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Update embryo.csv from txt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "If anything looks fishy at this stage, nothing changes in the txt files or the csv files (until the csv file is saved in the last cell).  \n",
    "So you can always rerun it from this point when the txt files are loaded."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(save_results_txt_path,'r') as f:\n",
    "    lines = f.read().split('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(fix_masks_txt_path,'r') as f:\n",
    "    lines1 = f.read().split('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "lines = list(filter(None, lines))\n",
    "lines1 = list(filter(None, lines1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "names = [l.split(',')[0] for l in lines]\n",
    "status = [l.split(',')[1] for l in lines]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "names_masks = [l.split(',')[0] for l in lines1]\n",
    "is_mask_bad = [l.split(',')[1] for l in lines1]\n",
    "\n",
    "# Take only bad masks\n",
    "bad_masks = [names_masks[i] for i,boo in enumerate(is_mask_bad) if boo=='True']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(900, 34)"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(names_masks),len(bad_masks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Yay! No doubles!!\n"
     ]
    }
   ],
   "source": [
    "# Check for doubles:\n",
    "\n",
    "if len(names) != len(set(names)):\n",
    "    print('WE GOT DOUBLES!! Taking \"status\" from last choice only')\n",
    "else:\n",
    "    print('Yay! No doubles!!')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Cross with masks list:\n",
    "## I expect that if \"fix mask\" was chosen then \"status\" should be 1 (good embryo)\n",
    "\n",
    "for n in bad_masks:\n",
    "    idx = names.index(n)\n",
    "    if status[idx] == 'False':\n",
    "        print(f'{n} is in the fix mask list but status is bad. Fixing Status to good')\n",
    "        status[idx] = 'True'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "csv_file = pd.read_csv(csv_path)\n",
    "csv_file = csv_file.reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Status - 1=True, 3=False (aka bad embryo)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i,n in enumerate(names):\n",
    "    idx = csv_file.index[csv_file['cropped_image_file'] == f'{n}.tif'].tolist()\n",
    "    if len(idx)!=1:\n",
    "        print(f'oh oh. found {len(idx)} csv rows for {n}')\n",
    "    else:\n",
    "        csv_file.at[idx[0],'status'] = 1 if status[i]=='True' else 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save csv\n",
    "\n",
    "csv_file.to_csv(csv_path, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### DONT FORGET TO COMMIT AND PUSH!!\n",
    "by runnning this in the \"embryos_csv\" dir:  \n",
    "```git add embryos.csv  \n",
    "git commit -m \"new embryos and masks verified\"  \n",
    "git push origin master  \n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}