CZI-S1.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "72431ca6-f28b-4268-82e2-38a058eb14ff",
   "metadata": {},
   "source": [
    "# Image Processing for CZI Files"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0c49a706-6a58-4a6b-8ec8-233a43e245bf",
   "metadata": {},
   "source": [
    "## Splitting the combined .czi file into separate channels"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d4b0fad7-3386-448e-8776-7e2203441bf1",
   "metadata": {},
   "source": [
    "### List of Tasks"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f076d1d7-cfc1-4a0f-8b63-2079ae6e02c9",
   "metadata": {},
   "source": [
    "1. Read CZI file from a given folder.\n",
    "2. Read metadata for each file and based on channel number split into separate channels.\n",
    "3. Save each channel as a separate tif/tiff file with correct channel name without permission errors."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "12bbda56-fc13-4dd4-a623-d46171a37741",
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import nrrd\n",
    "import numpy as np\n",
    "import os\n",
    "import cv2 as cv\n",
    "import tifffile as tiff\n",
    "from scipy import ndimage\n",
    "from numpy import ndarray\n",
    "from aicsimageio import AICSImage"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c939a731-1290-46fa-90ec-5bff25c704fa",
   "metadata": {},
   "source": [
    "### Enter the exact/full path of the folder where images are located."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "c7b76a95-7169-4d63-8267-ace80e12e767",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Enter the exact/full path of the folder where images are located.\n",
    "c_path = r\"D:\\LinesReg\\210720_itpr1b_GCaMP6s_new\"\n",
    "original_path = c_path + r\"\\original\"\n",
    "line_path = original_path\n",
    "processed_path = line_path + \"/processed/\"\n",
    "processed_for_average_path = line_path + \"/processed_for_average/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "93888a85-809b-4142-b790-dc32e8d731c2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating D:\\LinesReg\\210720_itpr1b_GCaMP6s_new\\original\n",
      "Creating D:\\LinesReg\\210720_itpr1b_GCaMP6s_new\\original/processed/\n",
      "Creating D:\\LinesReg\\210720_itpr1b_GCaMP6s_new\\original/processed_for_average/\n"
     ]
    }
   ],
   "source": [
    "if not os.path.exists(processed_path) or os.path.exists(processed_for_average_path) or os.path.exists(original_path):\n",
    "    print(f\"Creating {original_path}\")\n",
    "    os.makedirs(original_path, exist_ok=True)\n",
    "    print(f\"Creating {processed_path}\")\n",
    "    os.makedirs(processed_path, exist_ok=True)\n",
    "    print(f\"Creating {processed_for_average_path}\")\n",
    "    os.makedirs(processed_for_average_path, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "ebc441b8-f35f-4a6d-9a87-4cecd5135186",
   "metadata": {},
   "outputs": [
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Enter Reference Channel Number: 1\n"
     ]
    }
   ],
   "source": [
    "ref_ch_num = input(\"Enter Reference Channel Number:\")\n",
    "#Starts from 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "5efed57f-d581-4391-b788-6eef52df0a78",
   "metadata": {},
   "outputs": [],
   "source": [
    "def _rotate(src, angle):\n",
    "    # angle in degrees\n",
    "    rotated_matrix = ndimage.rotate(src, angle=angle, reshape=False)\n",
    "    return rotated_matrix\n",
    "\n",
    "\n",
    "def image_to_nrrd(image, image_name):\n",
    "    Header = {'units': ['m', 'm', 'm'], 'spacings': [voxel_width, voxel_height, 1e-6]}\n",
    "    print(f'Creating nrrd image with name : {image_name}.nrrd')\n",
    "    return nrrd.write(os.path.join(processed_path, f\"{image_name}.nrrd\"), image, header=Header, index_order='C')\n",
    "\n",
    "\n",
    "def image_to_tiff(image, image_name):\n",
    "    print(f'Creating file {image_name}.tif')\n",
    "    return tiff.imwrite(os.path.join(processed_for_average_path, f\"{image_name}.tif\"), image,\n",
    "                        metadata={'spacing': ['1./VoxelSizeList[0]', '1./VoxelSizeList[0]', '1'], 'unit': 'um',\n",
    "                        'axes ': 'ZYX', 'imagej': 'True'})\n",
    "\n",
    "\n",
    "def get_channel_name(f):\n",
    "    name, ext = f.split(\".\")\n",
    "    a, ref_ch_name = name.rsplit('_', 1)\n",
    "    sig_ch_name, fish_num = a.rsplit('_', 1)\n",
    "    return name, ref_ch_name, sig_ch_name, fish_num\n",
    "\n",
    "\n",
    "def get_image_data(f):\n",
    "    num_stacks, h, w = f.shape[3:]\n",
    "    # Determine voxel spacing - x, y for use later while writing nrrd files to be of correct pixel spacing. This info\n",
    "    # can be verified by in Fiji by [ image -> Properties]\n",
    "    voxel_x, voxel_y, voxel_z = f.get_physical_pixel_size()[:3]  # read_voxel_size(first_channel_data)\n",
    "    if voxel_z != 1e-6:\n",
    "        # Warning to user if voxel depth is being reset to 1micron, to be compatible with zebrafish pipeline\n",
    "        print(f\"Unsuitable voxel depth Value: {voxel_z}. Will be reset to : 1e-6.\")\n",
    "    return num_stacks, h, w, voxel_x, voxel_y, voxel_z\n",
    "\n",
    "\n",
    "def contrast_enhancement(f):\n",
    "    alpha = 10.0  # Contrast control (1.0-3.0) but 10 is required for my purposes here\n",
    "    beta = 5  # Brightness control (0-100). Not to be added beyond 5, to not hamper the signal with salt and pepper\n",
    "    # noise.\n",
    "    contrast_enhanced_image = cv.convertScaleAbs(f, alpha=alpha, beta=beta)\n",
    "    return contrast_enhanced_image.astype('uint8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e95fea36-f1db-41d4-8843-e3cbdbfc0173",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "itpr1b_1_GCaMP6s.czi\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\kesha\\anaconda3\\lib\\site-packages\\aicsimageio\\transforms.py:177: UserWarning: Data has dimension B with depth 1, assuming B=0 is the desired value, if not the case specify B=x where x is an integer, list, tuple, range, or slice.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Height, Width of image stack: 1946 , 1024 with 303 stacks!\n",
      "Voxel Details (x, y, depth): 4.151329120498975e-07 , 4.151329120498975e-07 , 1e-06\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\kesha\\anaconda3\\lib\\site-packages\\aicsimageio\\transforms.py:177: UserWarning: Data has dimension B with depth 1, assuming B=0 is the desired value, if not the case specify B=x where x is an integer, list, tuple, range, or slice.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Image stack to be rotated: itpr1b_1_GCaMP6s.czi\n"
     ]
    },
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Enter the angle by which image to be rotated: 1\n"
     ]
    }
   ],
   "source": [
    "for file in os.listdir(c_path):\n",
    "    if file.endswith(\".czi\"):\n",
    "        print(file)\n",
    "        line_name, reference_channel_name, signal_channel_name, fish_number = get_channel_name(file)\n",
    "\n",
    "        # Access the image data and obtain details (metadata) of the image\n",
    "        c = AICSImage(os.path.join(c_path, file))\n",
    "        N_stacks, height, width, voxel_width, voxel_height, voxel_depth = get_image_data(c)\n",
    "        print(\"Height, Width of image stack:\", height, \",\", width, f\"with {N_stacks} stacks!\")\n",
    "        print(\"Voxel Details (x, y, depth):\", voxel_width, \",\", voxel_height, \",\", voxel_depth)\n",
    "\n",
    "        # Obtain the image data from respective channels\n",
    "        if ref_ch_num == 0:\n",
    "            ref_channel_data = c.get_image_data(\"ZYX\", C=0, S=0, T=0)\n",
    "            sig_channel_data = c.get_image_data(\"ZYX\", C=1, S=0, T=0)\n",
    "\n",
    "        else:\n",
    "            ref_channel_data = c.get_image_data(\"ZYX\", C=1, S=0, T=0)\n",
    "            sig_channel_data = c.get_image_data(\"ZYX\", C=0, S=0, T=0)\n",
    "\n",
    "        ##***********Creating image by stacking the 2D matrix into a 3D Array****##\n",
    "        RImage: ndarray = np.stack(ref_channel_data).astype('uint8')\n",
    "        SImage: ndarray = np.stack(sig_channel_data).astype('uint8')\n",
    "\n",
    "        ##***********Contrast Enhancement*********************************##\n",
    "        CE_image_R = contrast_enhancement(RImage)\n",
    "        CE_image_S = contrast_enhancement(SImage)\n",
    "\n",
    "        ##***********Rotation*********************************##\n",
    "        print(f'Image stack to be rotated: {file}')\n",
    "\n",
    "        theta = float(input('Enter the angle by which image to be rotated:'))\n",
    "        Reference_image_Rotated = _rotate(CE_image_R, theta)\n",
    "        Signal_image_Rotated = _rotate(CE_image_S, theta)\n",
    "\n",
    "        # **********Final Saving of Images to respective folders********\n",
    "        print(\"Final Saving of Images to respective folders!\")\n",
    "        Reference_image_name = f'{signal_channel_name}_{fish_number}_{reference_channel_name}'\n",
    "        Signal_image_name = f'{signal_channel_name}_{fish_number}_GFP'\n",
    "\n",
    "        Reference_nrrd_image: nrrd = image_to_nrrd(RImage, Reference_image_name)\n",
    "        Reference_tiff_image: tiff = image_to_tiff(RImage, Reference_image_name)\n",
    "\n",
    "        Signal_nrrd_image: nrrd = image_to_nrrd(SImage, Signal_image_name)\n",
    "        Signal_tiff_image: tiff = image_to_tiff(SImage, Signal_image_name)\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a0d4672e-8e47-42e3-b495-3710b635bf37",
   "metadata": {},
   "source": [
    "# Creating slurmscript folder inside the location of raw data (c_path) as provided earlier.\n",
    "\n",
    "### Here, you will find all the slurmscripts that are needed for running computational tasks in the cluster. This folder is to be created for every line, by running the cells below."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "052519b9",
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "slurmscriptpath = c_path + r\"/slurmscript\"\n",
    "if not os.path.exists(processed_path):\n",
    "    print(f\"Creating {original_path}\")\n",
    "    os.makedirs(original_path, exist_ok=True)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Now that all the lines have been processed, take a look at these images in Fiji and select the fish you would like to select as the base fish."
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a1e6de6d-af05-4633-9a4b-cf43f72ac455",
   "metadata": {},
   "outputs": [],
   "source": [
    "#%run average_script_generator.py\n",
    "# Creating Averaging Script\n",
    "from average_script_generator import create_averaging_script\n",
    "base_fish_num = input(\"Enter base fish number you have chosen: \")\n",
    "create_averaging_script(slurmscriptpath, base_fish_num, reference_channel_name, signal_channel_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fcea61b1-b31c-46da-b56d-ef8d1501dfdb",
   "metadata": {},
   "outputs": [],
   "source": [
    "#%run lines_script_generator.py\n",
    "# Creating Lines Script\n",
    "from lines_script_generator import create_lines_script\n",
    "create_lines_script(slurmscriptpath, reference_channel_name, signal_channel_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "da3fe4a8-6a6f-40d8-af75-955ec5467195",
   "metadata": {},
   "outputs": [],
   "source": [
    "#%run standard_script_generator.py\n",
    "# Creating Standard Script\n",
    "from standard_script_generator import create_standard_script\n",
    "create_standard_script(slurmscriptpath, reference_channel_name, signal_channel_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "### At this stage, both the pre-processing of the image data and also creating the slurm scripts has been completed. Now, please transfer all of these pre-processed images and slurmscripts into the cluster.\n",
    "\n",
    "### The images needs to be transferred:\n",
    "\n",
    "### For averaging job (Location on cluster):\n",
    "### For scripts:   From c_path/slurmscripts TO raven/u/atlas_scripts/fixedorlive/.\n",
    "### For averaging images: From c_path/processed_for_average TO raven/ptmp/avg_templ/images/liveorfixed/Low_res  (must be .tiff files)\n",
    "### For individual lines images: From c_path/processed TO raven/ptmp/avg_templ/images/liveorfixed/high_res       (must be .nrrd files)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "# Command for running slurm scripts on putty (open a session on putty and go to respective atlas_scripts folder from terminal)\n",
    "\n",
    "### For averaging job: 'sbatch ANTs_average_linename.sh'\n",
    "### For lines job: 'sbatch --array=1-N ANTs_lines_linename.sh' (Replace N with total number of fishes)\n",
    "### For averaging job: 'sbatch ANTs_standard_linename.sh'"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "id": "9547244f-0709-4cc9-b60c-33142485202e",
   "metadata": {},
   "source": [
    "# Image Processing \n",
    "## ( Working with Images after downloading from Cluster)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c6efbc68-0d34-46d9-8310-46a20556c744",
   "metadata": {},
   "source": [
    "## Working with Average Images\n",
    "### Hint: these average images will be in the nifti (*.nii.gz) format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bfdfa01e-f2ff-46fa-8416-7861a5c28a5b",
   "metadata": {},
   "outputs": [],
   "source": [
    "%run AverageImageProcessing.py"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2057ba3d-5cf7-40af-9212-ee2376827b09",
   "metadata": {},
   "source": [
    "## Working with Individual Lines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b0021fac-b8f9-473c-b9d8-aabfe993cc56",
   "metadata": {},
   "outputs": [],
   "source": [
    "%run GeneralImageProcessing.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e742874c-9a1b-42af-80d6-21007f1288e6",
   "metadata": {},
   "outputs": [],
   "source": [
    "## Working with Transformed Lines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8e7fcc1b-8bce-46bd-98c6-936a6bb5a008",
   "metadata": {},
   "outputs": [],
   "source": [
    "%run LinesTransformationProcessing.py\n",
    "%run TransformationImageProcessing.py"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}