Comparison.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from PIL import Image, ImageFilter\n",
    "import math\n",
    "from os import listdir\n",
    "import random\n",
    "import cv2\n",
    "import pandas as pd\n",
    "\n",
    "import pywt\n",
    "\n",
    "import keras\n",
    "from keras import backend as K\n",
    "from keras.models import model_from_json\n",
    "from keras.models import Model, Sequential\n",
    "from keras.layers import Input, Convolution2D, Conv2D, MaxPooling2D, Dense, Dropout, Flatten, Conv2D, BatchNormalization, GlobalAveragePooling2D\n",
    "from keras.callbacks import LearningRateScheduler\n",
    "from keras.optimizers import SGD, Adam\n",
    "from keras import regularizers, optimizers, losses, initializers\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow.python.client import device_lib\n",
    "\n",
    "import skimage\n",
    "from skimage import measure\n",
    "\n",
    "from PIL import Image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from srcnn import *\n",
    "from utility import *\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Division by 4\n",
    "window_size= 64\n",
    "scale = 4\n",
    "stride = 16"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calcPSNR(original,prediction):\n",
    "    return measure.compare_psnr(original, prediction,data_range=255)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calcSSIM(orig,pred):\n",
    "    return measure.compare_ssim(orig,pred,data_range=255, multichannel=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calcRMSE(orig,pred):\n",
    "    return np.sqrt(((orig-pred)**2).mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Returns original, bicubic and model prediction\n",
    "def getImages(filename):\n",
    "    im = Image.open(\"set14/\"+filename)\n",
    "\n",
    "    img = np.asarray(im.convert('YCbCr'))\n",
    "\n",
    "    img = img[:,:,0]\n",
    "    org, bic = get_input_images(img, 4)\n",
    "\n",
    "    h, w = retSize(org)\n",
    "\n",
    "    sub_img = getSubImages(bic)\n",
    "    \n",
    "    low_w = []\n",
    "    for i in range(len(sub_img)):\n",
    "        dwt_w = pywt.dwt2(sub_img[i], 'haar')\n",
    "\n",
    "        dwt_w = np.asfarray([dwt_w[0], dwt_w[1][0], dwt_w[1][1], dwt_w[1][2]])\n",
    "\n",
    "        low_w.append(dwt_w)\n",
    "        \n",
    "    low_wavelet = np.asarray(low_w)\n",
    "    \n",
    "    pred_w = model.predict(low_wavelet)\n",
    "    \n",
    "    res = low_wavelet + pred_w\n",
    "\n",
    "    sub_imgs = []\n",
    "    for i in range(len(res)) :\n",
    "        patch = res[i]\n",
    "        dwt = (patch[0], (patch[1], patch[2], patch[3]))\n",
    "        wavelet = pywt.idwt2(dwt, 'haar')\n",
    "        sub_imgs.append(wavelet)\n",
    "        \n",
    "    im = patch_to_image(sub_imgs, h, w)\n",
    "    return org,bic,im"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def convertToYCbCr(x):\n",
    "    return [image.convert('YCbCr') for image in x]\n",
    "\t\n",
    "\t#Display images\n",
    "def print_luminance(img, ch=1) :\n",
    "    if (ch == 1) :\n",
    "        plt.imshow(img, cmap=plt.get_cmap('gray'), interpolation='nearest')\n",
    "    else :\n",
    "        plt.imshow(img[:,:,0], cmap=plt.get_cmap('gray'), interpolation='nearest')\n",
    "\t\t\n",
    "def getSubImage(img, height, width, centerX, centerY, len_interval, scale = 1) :\n",
    "    window = window_size // scale\n",
    "    return img[ height   : height//2 + window + centerX*len_interval,\n",
    "                width // 2  + centerY*len_interval : width//2 + window + centerY*len_interval : ]\n",
    "\t\t\t\t\n",
    "def retSize(x):\n",
    "    return (x.shape[0], x.shape[1])\n",
    "\t\n",
    "def getSubImages(image, stride = stride) :\n",
    "    sub = []\n",
    "    height, width = retSize(image)\n",
    "    for h in range(0, height - window_size, stride) :\n",
    "        for w in range (0, width - window_size, stride) : \n",
    "            sub.append(image[h : h + window_size, w : w + window_size])\n",
    "    return sub\n",
    "\n",
    "def appendSubbands(l1,l2,l3,l4, dwt):\n",
    "    l1 = np.append(l1 , dwt[0])\n",
    "    l2 = np.append(l2, dwt[1][0])\n",
    "    l3 = np.append(l3, dwt[1][1])\n",
    "    l4 = np.append(l4, dwt[1][2])\n",
    "    return (l1,l2,l3,l4)\n",
    "\n",
    "def reshape(matrix, dwt_shape, channel='channels_last', ch=1) :\n",
    "    if (channel == 'channels_last') :\n",
    "        return matrix.reshape(-1, dwt_shape[0], dwt_shape[1], ch)\n",
    "    else :\n",
    "        return matrix.reshape(ch, -1, dwt_shape[0], dwt_shape[1])\n",
    "\n",
    "def equals(a, b) :\n",
    "    t = a == b\n",
    "    count = 0\n",
    "    for a in range(0, len(t)) :\n",
    "        for b in range(len(t[0])) :\n",
    "            for c in range(len(t[1])) :\n",
    "                if ( t[a, b, c] == False):\n",
    "                    count = count + 1\n",
    "    return count / (32*32*4) < 0.20\n",
    "    \n",
    "\n",
    "def getSubImages(image, stride = stride) :\n",
    "    sub = []\n",
    "    height, width = retSize(image)\n",
    "    for h in range(0, height - window_size, stride) :\n",
    "        for w in range (0, width - window_size, stride) : \n",
    "            sub.append(image[h : h + window_size, w : w + window_size])\n",
    "    return sub\n",
    "\n",
    "def plot4images(im1, im2, im3, im4, ch=1) :\n",
    "    size = (10, 10)\n",
    "    fig = plt.figure(figsize=size)\n",
    "    \n",
    "    plt.subplot(221)\n",
    "    plt.title(\"Approximation\")\n",
    "    print_luminance(im1, ch )\n",
    "    plt.subplot(222)\n",
    "    plt.title(\"Horizontal\")\n",
    "    print_luminance(im2, ch)\n",
    "    plt.subplot(223)\n",
    "    plt.title(\"Vertical\")\n",
    "    print_luminance(im3, ch)\n",
    "    \n",
    "    plt.subplot(224)\n",
    "    plt.title(\"Diagonal\")\n",
    "    print_luminance(im4, ch)\n",
    "\n",
    "def plot_waveletTrans(wt, ch=1) :\n",
    "    plot4images(wt[0], wt[1][0], wt[1][1], wt[1][2], ch)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "PATH = \"\"\n",
    "TRAIN_FILE = PATH + \"set14/\"\n",
    "obj_files = sorted(listdir(TRAIN_FILE )) #the filenames  the call to sorted is actually necessary to ensure independence to filesystems"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "high_res=obj_files[0::2]\n",
    "low_res=obj_files[1::2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [],
   "source": [
    "space_grey = load_model(\"sptial_deep_model_50_ycbcr\")\n",
    "wave_grey = load_model(\"SRCNN_ycbcr_50_residual\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 227,
   "metadata": {},
   "outputs": [],
   "source": [
    "def reconstructSpatialImage(model, bic):\n",
    "    height, width = retSize(bic)\n",
    "    padding = 100\n",
    "    padded_img = np.pad(bic, pad_width = padding, mode='symmetric')\n",
    "\n",
    "    space_img = np.zeros((height,width))\n",
    "    stride = 4\n",
    "    window_size = 32\n",
    "    half_window = window_size//2\n",
    "    for h in range(padding, padded_img.shape[0]-padding, stride):\n",
    "        h_l = h - padding\n",
    "        h_h = min(height, h-padding+1+stride)\n",
    "        for w in range(padding, padded_img.shape[1]-padding, stride):\n",
    "            d = np.reshape(padded_img[h-half_window:h+half_window, \n",
    "                                      w-half_window:w+half_window], (window_size,window_size,1))\n",
    "            window = np.reshape(model.predict(np.expand_dims(d,axis=0)/255., 1),(window_size,window_size)) #+ padded_img[h-half_window:h+half_window, w-half_window:w+half_window]/255.\n",
    "            w_h = min(width, w-padding+1+stride)\n",
    "            w_l = w - padding\n",
    "            space_img[h_l:h_h, w_l:w_h] = window[half_window:half_window+h_h-h_l,\n",
    "                                                 half_window:half_window+w_h-w_l]\n",
    "            del d\n",
    "            del window\n",
    "    del padded_img\n",
    "    return space_img"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 228,
   "metadata": {},
   "outputs": [],
   "source": [
    "def reconstructWaveletImage(model, bic):\n",
    "    height, width = retSize(bic)\n",
    "    padding = 100\n",
    "    padded_img = np.pad(bic, pad_width = padding, mode='symmetric')\n",
    "\n",
    "    wave_image = np.zeros((height,width))\n",
    "    stride = 4\n",
    "    window_size = 64\n",
    "    half_window = window_size//2\n",
    "    for h in range(padding, padded_img.shape[0]-padding, stride):\n",
    "        h_l = h - padding\n",
    "        h_h = min(height, h-padding+1+stride)\n",
    "        for w in range(padding, padded_img.shape[1]-padding, stride):\n",
    "            d = padded_img[h-half_window:h+half_window, \n",
    "                                      w-half_window:w+half_window]\n",
    "            dwt_w = pywt.dwt2(d, 'haar')\n",
    "            dwt_w = np.asfarray([dwt_w[0], dwt_w[1][0], dwt_w[1][1], dwt_w[1][2]])\n",
    "            patch = model.predict(np.expand_dims(dwt_w,axis=0)/255., 1)\n",
    "            dwt = (patch[0][0], (patch[0][1], patch[0][2], patch[0][3]))\n",
    "            window = pywt.idwt2(dwt, 'haar')\n",
    "            w_h = min(width, w-padding+1+stride)\n",
    "            w_l = w - padding\n",
    "            wave_image[h_l:h_h, w_l:w_h] = window[half_window:half_window+h_h-h_l,\n",
    "                                                 half_window:half_window+w_h-w_l]\n",
    "            del d\n",
    "            del window\n",
    "            del dwt_w\n",
    "            del patch\n",
    "            del dwt\n",
    "    del padded_img\n",
    "    return wave_image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 229,
   "metadata": {},
   "outputs": [],
   "source": [
    "def reconstructWaveAndSpaceImage(img, model_wave, model_space):\n",
    "    org, bic = get_input_images(img, 4.)\n",
    "    space = reconstructSpatialImage(model_space, bic)\n",
    "    print(\"done with space\")\n",
    "    wave = reconstructWaveletImage(model_wave, bic)\n",
    "    print(\"done with freq\")\n",
    "    return space, wave, org, bic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 247,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getPredictions(path, model_wave, model_space):\n",
    "    img = Image.open(\"set14/\"+path)\n",
    "    img = np.asarray(img.convert('YCbCr'))[:,:,0]\n",
    "    return reconstructWaveAndSpaceImage(img, wave_grey, space_grey)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 248,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "img_001_SRF_4_HR.png\n",
      "done with space\n",
      "done with freq\n",
      "img_002_SRF_4_HR.png\n",
      "done with space\n",
      "done with freq\n",
      "img_003_SRF_4_HR.png\n",
      "done with space\n",
      "done with freq\n",
      "img_004_SRF_4_HR.png\n",
      "done with space\n",
      "done with freq\n",
      "img_005_SRF_4_HR.png\n",
      "done with space\n",
      "done with freq\n",
      "img_006_SRF_4_HR.png\n",
      "done with space\n",
      "done with freq\n",
      "img_007_SRF_4_HR.png\n",
      "done with space\n",
      "done with freq\n",
      "img_008_SRF_4_HR.png\n",
      "done with space\n",
      "done with freq\n",
      "img_009_SRF_4_HR.png\n",
      "done with space\n",
      "done with freq\n",
      "img_010_SRF_4_HR.png\n",
      "done with space\n",
      "done with freq\n",
      "img_011_SRF_4_HR.png\n",
      "done with space\n",
      "done with freq\n",
      "img_012_SRF_4_HR.png\n",
      "done with space\n",
      "done with freq\n",
      "img_013_SRF_4_HR.png\n",
      "done with space\n",
      "done with freq\n",
      "img_014_SRF_4_HR.png\n",
      "done with space\n",
      "done with freq\n"
     ]
    }
   ],
   "source": [
    "results=[]\n",
    "\n",
    "for f in high_res:\n",
    "    print(f)\n",
    "    space_pred, wave_pred, org, bic = getPredictions(f, wave_grey, space_grey)\n",
    "    results.append((org,bic,wave_pred, space_pred))\n",
    "    \n",
    "np.save(\"results\", results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 249,
   "metadata": {},
   "outputs": [],
   "source": [
    "results=np.load(\"results.npy\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Methods to crunch numbers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 250,
   "metadata": {},
   "outputs": [],
   "source": [
    "def convertToPicture(img):\n",
    "    if(len(img.shape) == 3):\n",
    "        img = Image.fromarray(img, 'RGB')\n",
    "    else:\n",
    "        img = Image.fromarray(img)\n",
    "    if img.mode != 'RGB':\n",
    "        img = img.convert('RGB')\n",
    "    return img"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 251,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_results(i):\n",
    "    result=results[i]\n",
    "    org=result[0].astype(float)\n",
    "    bic=result[1].astype(float)\n",
    "    sr=result[2].astype(float)\n",
    "    spred=result[3].astype(float)\n",
    "    \n",
    "    psnr_bic=round(calcPSNR(org,bic),4)\n",
    "    psnr_sr=round(calcPSNR(org,sr),4)\n",
    "    psnr_sd=round(calcPSNR(org, spred),4)\n",
    "    org,bic,sr, spred = ims[i]\n",
    "    \n",
    "    plot_images([org,bic,sr, spred], [\"Original\", \"Bic (PSNR: \"+str(psnr_bic)+\")\", \"SR (PSNR: \"+str(psnr_sr)+\")\", \"Spatial (PSNR: \"+str(psnr_sd)+\")\"], size= (20,20), ch=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 252,
   "metadata": {},
   "outputs": [],
   "source": [
    "def displayTab(t, cols, indexValues):\n",
    "    df=pd.DataFrame(data=t)\n",
    "    df.columns=cols\n",
    "    df.index=indexValues\n",
    "    return df\n",
    "\n",
    "def waveletTransf(image):\n",
    "    dwt_w = pywt.dwt2(image, wavelet='haar')\n",
    "    return np.asfarray([dwt_w[0], dwt_w[1][0], dwt_w[1][1], dwt_w[1][2]])\n",
    "\n",
    "def waveDomainMeasure(image, obj, measure):\n",
    "    dwt_image = waveletTransf(image)\n",
    "    dwt_obj = waveletTransf(obj)\n",
    "    if measure == 'ssim':\n",
    "        op = calcSSIM\n",
    "    if measure == 'rmse':\n",
    "        op = calcRMSE\n",
    "    return (op(dwt_obj[0],dwt_image[0]),\n",
    "            op(dwt_obj[1],dwt_image[1]),\n",
    "            op(dwt_obj[2],dwt_image[2]),\n",
    "            op(dwt_obj[3],dwt_image[3]))\n",
    "        \n",
    "def waveMeasureWaveletAndSpatial(bic, image_space, image_wave, obj, measure):\n",
    "    return (waveDomainMeasure(bic, obj, measure),\n",
    "            waveDomainMeasure(image_space, obj, measure), \n",
    "            waveDomainMeasure(image_wave, obj, measure))\n",
    "\n",
    "\"\"\"Returns mean measure of the dataset result (assumed of the form [original, bicubic, wavelet, spatial])\n",
    "for each wavelet frequency, between spatial image and original and wavelet with original.\n",
    "Returns the result in the order space, wave\"\"\"\n",
    "def averageWaveMeasure(results, measure, f):\n",
    "    waveAndSpace = []\n",
    "    for res in results:\n",
    "        waveAndSpace.append(waveMeasureWaveletAndSpatial(res[1], \n",
    "                                                         f(res[3]), \n",
    "                                                         f(res[2]), \n",
    "                                                         res[0], measure)) # remember the order: original, bicubic, wavelet, spatial\n",
    "    return np.mean(waveAndSpace, axis=0)\n",
    "\n",
    "\"\"\"Returns mean measure of the dataset result (assumed of the form [original, bicubic, wavelet, spatial])\n",
    "taken in space, between spatial net image and original and wavelet net with original.\n",
    "Returns the result in the order space, wave\"\"\"\n",
    "def averageSpaceMeasure(results, measure, f):\n",
    "    waveAndSpace= []\n",
    "    if measure == 'psnr':\n",
    "        op = calcPSNR\n",
    "    if measure == 'rmse':\n",
    "        op = calcRMSE\n",
    "    if measure == 'ssim':\n",
    "        op = calcSSIM\n",
    "    for res in results:\n",
    "        waveAndSpace.append((op(res[0], res[1]),\n",
    "                             op(res[0], f(res[3])), \n",
    "                             op(res[0], f(res[2]))))\n",
    "    return np.mean(waveAndSpace, axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 257,
   "metadata": {},
   "outputs": [],
   "source": [
    "def displayMeasureResults(results,f):\n",
    "    t1 = pd.concat([displayTab(averageWaveMeasure(results, 'ssim', f), \n",
    "                      [\"mean LL Band SSIM\", \"mean LH Band SSIM\", \"mean HL Band SSIM\", \"mean HH Band SSIM\"], \n",
    "           [\"Bicubic\",\"Spatial SRCNN\", \"Wavelet SRCNN\"]),\n",
    "           displayTab(averageWaveMeasure(results, 'rmse', f), \n",
    "                      [\"mean LL Band RMSE\", \"mean LH Band RMSE\", \"mean HL Band RMSE\", \"mean HH Band RMSE\"], \n",
    "           [\"Bicubic\",\"Spatial SRCNN\", \"Wavelet SRCNN\"])], axis=1)\n",
    "    t2 = pd.concat([displayTab(averageSpaceMeasure(results, 'psnr', f), [\"mean PSNR\"], [\"Bicubic\", \"Spatial SRCNN\", \"Wavelet SRCNN\"]),\n",
    "          displayTab(averageSpaceMeasure(results, 'rmse',f), [\"mean RMSE\"], [\"Bicubic\",\"Spatial SRCNN\", \"Wavelet SRCNN\"]),\n",
    "          displayTab(averageSpaceMeasure(results, 'ssim',f), [\"mean SSIM\"], [\"Bicubic\",\"Spatial SRCNN\", \"Wavelet SRCNN\"])],axis=1)\n",
    "    return pd.concat([t1,t2], axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Clipping impact on SSIM and RMSE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 258,
   "metadata": {},
   "outputs": [],
   "source": [
    "def convertNoClip(x):\n",
    "    return np.uint8(x*255)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 259,
   "metadata": {},
   "outputs": [],
   "source": [
    "def convertClip(x):\n",
    "    return np.uint8(np.clip(x,0,1)*255)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 260,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean LL Band SSIM</th>\n",
       "      <th>mean LH Band SSIM</th>\n",
       "      <th>mean HL Band SSIM</th>\n",
       "      <th>mean HH Band SSIM</th>\n",
       "      <th>mean LL Band RMSE</th>\n",
       "      <th>mean LH Band RMSE</th>\n",
       "      <th>mean HL Band RMSE</th>\n",
       "      <th>mean HH Band RMSE</th>\n",
       "      <th>mean PSNR</th>\n",
       "      <th>mean RMSE</th>\n",
       "      <th>mean SSIM</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Bicubic</th>\n",
       "      <td>0.704350</td>\n",
       "      <td>0.534244</td>\n",
       "      <td>0.490209</td>\n",
       "      <td>0.618344</td>\n",
       "      <td>29.992380</td>\n",
       "      <td>13.816186</td>\n",
       "      <td>13.451690</td>\n",
       "      <td>6.976369</td>\n",
       "      <td>23.389095</td>\n",
       "      <td>7.027991</td>\n",
       "      <td>0.667652</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Spatial SRCNN</th>\n",
       "      <td>0.718573</td>\n",
       "      <td>0.554940</td>\n",
       "      <td>0.516398</td>\n",
       "      <td>0.628458</td>\n",
       "      <td>29.486667</td>\n",
       "      <td>14.585224</td>\n",
       "      <td>14.454077</td>\n",
       "      <td>8.132235</td>\n",
       "      <td>23.572869</td>\n",
       "      <td>6.786253</td>\n",
       "      <td>0.688161</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Wavelet SRCNN</th>\n",
       "      <td>0.719035</td>\n",
       "      <td>0.554807</td>\n",
       "      <td>0.517135</td>\n",
       "      <td>0.625497</td>\n",
       "      <td>29.037812</td>\n",
       "      <td>14.707280</td>\n",
       "      <td>13.958787</td>\n",
       "      <td>8.458644</td>\n",
       "      <td>23.611924</td>\n",
       "      <td>6.759918</td>\n",
       "      <td>0.688812</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               mean LL Band SSIM  mean LH Band SSIM  mean HL Band SSIM  \\\n",
       "Bicubic                 0.704350           0.534244           0.490209   \n",
       "Spatial SRCNN           0.718573           0.554940           0.516398   \n",
       "Wavelet SRCNN           0.719035           0.554807           0.517135   \n",
       "\n",
       "               mean HH Band SSIM  mean LL Band RMSE  mean LH Band RMSE  \\\n",
       "Bicubic                 0.618344          29.992380          13.816186   \n",
       "Spatial SRCNN           0.628458          29.486667          14.585224   \n",
       "Wavelet SRCNN           0.625497          29.037812          14.707280   \n",
       "\n",
       "               mean HL Band RMSE  mean HH Band RMSE  mean PSNR  mean RMSE  \\\n",
       "Bicubic                13.451690           6.976369  23.389095   7.027991   \n",
       "Spatial SRCNN          14.454077           8.132235  23.572869   6.786253   \n",
       "Wavelet SRCNN          13.958787           8.458644  23.611924   6.759918   \n",
       "\n",
       "               mean SSIM  \n",
       "Bicubic         0.667652  \n",
       "Spatial SRCNN   0.688161  \n",
       "Wavelet SRCNN   0.688812  "
      ]
     },
     "execution_count": 260,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "displayMeasureResults(results, convertNoClip)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 261,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean LL Band SSIM</th>\n",
       "      <th>mean LH Band SSIM</th>\n",
       "      <th>mean HL Band SSIM</th>\n",
       "      <th>mean HH Band SSIM</th>\n",
       "      <th>mean LL Band RMSE</th>\n",
       "      <th>mean LH Band RMSE</th>\n",
       "      <th>mean HL Band RMSE</th>\n",
       "      <th>mean HH Band RMSE</th>\n",
       "      <th>mean PSNR</th>\n",
       "      <th>mean RMSE</th>\n",
       "      <th>mean SSIM</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Bicubic</th>\n",
       "      <td>0.704350</td>\n",
       "      <td>0.534244</td>\n",
       "      <td>0.490209</td>\n",
       "      <td>0.618344</td>\n",
       "      <td>29.992380</td>\n",
       "      <td>13.816186</td>\n",
       "      <td>13.451690</td>\n",
       "      <td>6.976369</td>\n",
       "      <td>23.389095</td>\n",
       "      <td>7.027991</td>\n",
       "      <td>0.667652</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Spatial SRCNN</th>\n",
       "      <td>0.723667</td>\n",
       "      <td>0.561043</td>\n",
       "      <td>0.521837</td>\n",
       "      <td>0.633870</td>\n",
       "      <td>27.099446</td>\n",
       "      <td>13.057959</td>\n",
       "      <td>12.841793</td>\n",
       "      <td>6.759755</td>\n",
       "      <td>24.150642</td>\n",
       "      <td>6.783948</td>\n",
       "      <td>0.691636</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Wavelet SRCNN</th>\n",
       "      <td>0.723896</td>\n",
       "      <td>0.561452</td>\n",
       "      <td>0.522789</td>\n",
       "      <td>0.631493</td>\n",
       "      <td>26.964752</td>\n",
       "      <td>13.046753</td>\n",
       "      <td>12.675799</td>\n",
       "      <td>6.777250</td>\n",
       "      <td>24.198652</td>\n",
       "      <td>6.755819</td>\n",
       "      <td>0.692389</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               mean LL Band SSIM  mean LH Band SSIM  mean HL Band SSIM  \\\n",
       "Bicubic                 0.704350           0.534244           0.490209   \n",
       "Spatial SRCNN           0.723667           0.561043           0.521837   \n",
       "Wavelet SRCNN           0.723896           0.561452           0.522789   \n",
       "\n",
       "               mean HH Band SSIM  mean LL Band RMSE  mean LH Band RMSE  \\\n",
       "Bicubic                 0.618344          29.992380          13.816186   \n",
       "Spatial SRCNN           0.633870          27.099446          13.057959   \n",
       "Wavelet SRCNN           0.631493          26.964752          13.046753   \n",
       "\n",
       "               mean HL Band RMSE  mean HH Band RMSE  mean PSNR  mean RMSE  \\\n",
       "Bicubic                13.451690           6.976369  23.389095   7.027991   \n",
       "Spatial SRCNN          12.841793           6.759755  24.150642   6.783948   \n",
       "Wavelet SRCNN          12.675799           6.777250  24.198652   6.755819   \n",
       "\n",
       "               mean SSIM  \n",
       "Bicubic         0.667652  \n",
       "Spatial SRCNN   0.691636  \n",
       "Wavelet SRCNN   0.692389  "
      ]
     },
     "execution_count": 261,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "displayMeasureResults(results, convertClip)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can notice here that clipping actually has a positive impact on the quality of the produced images, on all metrics. In general, the wavelet outperforms the spatial by a small margin in every domain and metric. In turn, the spatial outperforms the bicubic."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 262,
   "metadata": {},
   "outputs": [],
   "source": [
    "i=0\n",
    "ims = []\n",
    "for res in results:\n",
    "    im_o= convertToPicture(res[0])\n",
    "    im_w = convertToPicture(convertClip(res[2]))\n",
    "    im_b = convertToPicture(res[1])\n",
    "    im_s = convertToPicture(convertClip(res[3]))\n",
    "    im_o.save(\"srcnn_results/\"+str(i)+\"_original.png\")\n",
    "    im_w.save(\"srcnn_results/\"+str(i)+\"_wave.png\")\n",
    "    im_b.save(\"srcnn_results/\"+str(i)+\"_bic.png\")\n",
    "    im_s.save(\"srcnn_results/\"+str(i)+\"_space.png\")\n",
    "    i=i+1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The wavelet net outperforms the spatial net on all wavelet bands, which provide significant clue as to why images produced by the wavelet net actually seem better."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calculate_measures(measure):\n",
    "    bic_values=[]\n",
    "    sr_values=[]\n",
    "    spatial_values=[]\n",
    "\n",
    "    indexValues=[]\n",
    "    \n",
    "    cols=[\"Bicubic\",\"SRCNN\",\"Spatial SRCNN\"]\n",
    "    cols=[x+\" \"+measure for x in cols]\n",
    "\n",
    "    i=1\n",
    "    for result in results:\n",
    "        org=result[0].astype(float)\n",
    "        bic=result[1].astype(float)\n",
    "        sr=result[2].astype(float)\n",
    "        spatial=result[3].astype(float)\n",
    "        \n",
    "        if measure==\"psnr\":\n",
    "            bic_values.append(round(calcPSNR(org,bic),4))\n",
    "            sr_values.append(round(calcPSNR(org,sr),4))\n",
    "            spatial_values.append(round(calcPSNR(org,spatial),4))\n",
    "\n",
    "        if measure==\"rmse\":\n",
    "            bic_values.append(round(calcRMSE(org,bic),4))\n",
    "            sr_values.append(round(calcRMSE(org,sr),4))\n",
    "            spatial_values.append(round(calcRMSE(org,spatial),4))\n",
    "\n",
    "        if measure==\"ssim\":\n",
    "            bic_values.append(round(calcSSIM(org,bic),4))\n",
    "            sr_values.append(round(calcSSIM(org,sr),4))\n",
    "            spatial_values.append(round(calcSSIM(org,spatial),4))\n",
    "            \n",
    "        indexValues.append(\"Image \"+str(i))\n",
    "        i=i+1\n",
    "        \n",
    "    df=pd.DataFrame(data=[bic_values,sr_values,spatial_values]).transpose()\n",
    "    df.columns=cols\n",
    "    df.index=indexValues\n",
    "    \n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plotMeanStd(df,measure):\n",
    "    mean_values=[]\n",
    "    std_values=[]\n",
    "    \n",
    "    mean_values.append(df[\"Bicubic \"+measure].mean())\n",
    "    mean_values.append(df[\"SRCNN \"+measure].mean())\n",
    "    mean_values.append(df[\"Spatial SRCNN \"+measure].mean())\n",
    "    \n",
    "    std_values.append(df[\"Bicubic \"+measure].std())\n",
    "    std_values.append(df[\"SRCNN \"+measure].std())\n",
    "    std_values.append(df[\"Spatial SRCNN \"+measure].std())\n",
    "    \n",
    "    \n",
    "    xticks=['Bic','SR','SR+Spatial']\n",
    "    x = np.array([1, 2, 3])\n",
    "    \n",
    "    plt.xticks(x,xticks)\n",
    "    plt.errorbar(x, mean_values, std_values, linestyle='None', ecolor=\"red\",marker='o')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Will find values that are greater than 1 std away from mean\n",
    "def find_outliers(df,col):\n",
    "    mean=df[col].mean()\n",
    "    std=df[col].std()\n",
    "    return df[(df[col]>mean+std) | (df[col]<mean-std)][col]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>PSNR</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_psnr=calculate_measures(\"psnr\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_psnr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(df_psnr.idxmin())\n",
    "print(\"\")\n",
    "print(df_psnr.idxmax())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Lowest and highest PSNR was from image 1 and 6 respectively. Note that the lowest/highest value is consistent regardless\n",
    "of method. We can find the images that either performed rather well or poorly by looking at how much they deviate from\n",
    "mean (here looking at mean +- 1 std)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "find_outliers(df_psnr,\"Bicubic psnr\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "find_outliers(df_psnr,\"SRCNN psnr\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "find_outliers(df_psnr,\"Spatial SRCNN psnr\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For all 3 methods, it was the same set of images that deviates a lot.<br>\n",
    "Low measure: Image 1, 5, 13 <br>\n",
    "High measure: Image 6, 9"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Plot of mean and standard deviation for all 3 methods "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plotMeanStd(df_psnr,\"psnr\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>RMSE</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_rmse=calculate_measures(\"rmse\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_rmse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(df_rmse.idxmin())\n",
    "print(\"\")\n",
    "print(df_rmse.idxmax())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "find_outliers(df_rmse,\"Bicubic rmse\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "find_outliers(df_rmse,\"SRCNN rmse\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "find_outliers(df_rmse,\"Spatial SRCNN rmse\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plotMeanStd(df_rmse,\"rmse\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>SSIM</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_ssim=calculate_measures(\"ssim\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_ssim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(df_ssim.idxmin())\n",
    "print(\"\")\n",
    "print(df_ssim.idxmax())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "find_outliers(df_ssim,\"Bicubic ssim\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "find_outliers(df_ssim,\"SRCNN ssim\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "find_outliers(df_ssim,\"Spatial SRCNN ssim\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plotMeanStd(df_ssim,\"ssim\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "PSNR and RMSE behave similarly in regards to which images where super resolved the best and worst and which ones. SSIM on the other hand has a different subset of images with measures far from mean."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Plots</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(14):\n",
    "    plot_results(i)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}