SNe_shift_waveform-K-fold-nostrangewaves.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from numpy.random import seed\n",
    "seed(1337)\n",
    "from tensorflow import set_random_seed\n",
    "set_random_seed(2674)\n",
    "import numpy as np\n",
    "import h5py\n",
    "import matplotlib\n",
    "matplotlib.use('Qt4Agg')\n",
    "import matplotlib.pyplot as plt\n",
    "from scipy import integrate, interpolate, signal, optimize, stats\n",
    "import cPickle as pickle\n",
    "import lal\n",
    "import keras\n",
    "from keras.models import Sequential, load_model\n",
    "from keras.layers import Dense, Conv2D, MaxPool2D, Dropout, BatchNormalization, Flatten\n",
    "from keras.optimizers import Nadam, SGD\n",
    "from keras.callbacks import ModelCheckpoint\n",
    "from sklearn.utils import shuffle\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "import pyfftw\n",
    "import progressbar\n",
    "import time\n",
    "from sklearn import metrics\n",
    "import itertools\n",
    "np.set_printoptions(edgeitems=30, linewidth=160)\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This code is for reading simulated SNe waveforms\n",
    "# This code will apply shift to the waveform \n",
    "# samples so that the waveform will always be in the certre +- user customized percentage."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The name of the file that contains the simulated CCSN waveforms\n",
    "filename = './Data/GWdatabase.h5'\n",
    "\n",
    "# Read the simulated CCSN waveforms\n",
    "waveformfile = h5py.File(filename, 'r')\n",
    "\n",
    "\n",
    "# The first level keys of the h5 file\n",
    "reduced_data = waveformfile.keys()[0]\n",
    "waveformfilekey = waveformfile.keys()[1]\n",
    "yeofrho = waveformfile.keys()[2]\n",
    "\n",
    "waveformfamily = []\n",
    "waveformfamily_keys = []\n",
    "\n",
    "# Since there are 1824 different simulated CCSN waveform. \n",
    "# Each of which is saved in a different waveformfile key \n",
    "# So the loop below is to retreive all the keys with which the waveform strain data is accessed,\n",
    "# and save it to waveformfamily.\n",
    "# Each waveform family has 5 different keys, so the second part is to retrieve these 5 keys, and save them\n",
    "# to waveformfamily_keys.\n",
    "\n",
    "for i, key in enumerate(waveformfile[waveformfilekey].keys()):\n",
    "    waveformfamily.append(key)\n",
    "    if i == 0:\n",
    "        for j, _ in enumerate(waveformfile[waveformfilekey][waveformfamily[i]].keys()):\n",
    "            waveformfamily_keys.append(waveformfile[waveformfilekey][waveformfamily[i]].keys()[j])\n",
    "originalSNR = np.array(waveformfile[reduced_data][u'SNR(aLIGOfrom10kpc)'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(56839, 670, 13156, 1416)\n"
     ]
    }
   ],
   "source": [
    "# This is to set some parameters for the training.\n",
    "# Since the waveforms are stored in the unit of strain * distan\n",
    "# the waveform amplitudes need to be divided by a distance.\n",
    "\n",
    "# Convection factor between par sec and meters\n",
    "PctMe = lal.PC_SI\n",
    "\n",
    "# The distance the waveform will be divided by, in centimeters\n",
    "Dist = 10.0 * 1e3 * PctMe * 1e2\n",
    "\n",
    "# Since the waveform samples come in different lengths, \n",
    "# so every waveform sample will be set to the longest length.\n",
    "# findmax/findmin is a variable that saves the longest/shortest length of the waveform samples.\n",
    "# k/kmin is the index referring to the longest/shortest waveform sample.\n",
    "findmax = 0\n",
    "k = 0 \n",
    "findmin = 1e10\n",
    "kmin = 0\n",
    "#length = np.zeros(len(waveformfamily))\n",
    "#waveformfamily = [waveformfamily[0]]\n",
    "with open(\"abnormalwaves.pkl\") as f:\n",
    "    mark_nonzeros, mark_suddenjump = pickle.load(f)\n",
    "abn_index = np.concatenate([mark_nonzeros, mark_suddenjump])    \n",
    "\n",
    "# Since the waveform contains 1824 waveforms, which are different both in the morophology and the duration,\n",
    "# training a network with all these waveforms may make it hard to debug. So one may want to limit the variation\n",
    "# in the waveform samples by limiting the number of waveform samples put in the training. \n",
    "no_waves_considered = 1824\n",
    "#allwave = []\n",
    "#mark_nonzero=[]\n",
    "for i in range(len(waveformfamily[0:no_waves_considered])):\n",
    "    if i in abn_index:\n",
    "        continue\n",
    "    waveformnumber = i\n",
    "\n",
    "    ts = np.array(waveformfile[waveformfilekey][waveformfamily[waveformnumber]][u't-tb(s)']) \n",
    "    #waves = np.array(waveformfile[waveformfilekey][waveformfamily[waveformnumber]][u'strain*dist(cm)']) / Dist \n",
    "    #allwave.append(waves)\n",
    "    if findmax < len(ts):\n",
    "        findmax = len(ts)\n",
    "        k = i\n",
    "    if findmin > len(ts):\n",
    "        findmin = len(ts)\n",
    "        kmin = i\n",
    "\n",
    "print(findmax, k, findmin, kmin)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def rescale_to_set_SNR_single_waveform(preset_SNR, original_waveform, dt, Det):\n",
    "    \n",
    "    ds = len(original_waveform) * dt\n",
    "    df = 1.0 / ds\n",
    "    \n",
    "    Nf = int((len(original_waveform) // 2 + 1))\n",
    "    fs = np.arange(Nf) * df   \n",
    "    \n",
    "    # Amplitude spectral density\n",
    "    ASD = readnos(Det, fs)\n",
    "    \n",
    "    fftinput_for_snr = pyfftw.empty_aligned(len(original_waveform), dtype='complex128')     \n",
    "    fft_object_for_snr = pyfftw.builders.rfft(fftinput_for_snr)      \n",
    "    \n",
    "    wave_f = fft_object_for_snr(original_waveform) * dt\n",
    "    \n",
    "    temporary_snr = np.sqrt( 4.0 * sum( abs(wave_f) ** 2 / ASD ** 2 ) * df )\n",
    "    SNR_factor = preset_SNR / temporary_snr\n",
    "\n",
    "    wave = SNR_factor * original_waveform\n",
    "    return wave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ### This part is for finding \"unnatural/abnomral waves\"\n",
    "# mark_nonzeros = []\n",
    "# mark_suddenjump = []\n",
    "\n",
    "# wave = []\n",
    "# for count, i in enumerate(allwave):\n",
    "#     wave.append(rescale_to_set_SNR_single_waveform(10, i, 1.0/65535, 'H1'))\n",
    "#     if abs(wave[count][0]) > 0.2e-21:\n",
    "#         mark_nonzeros.append(count)\n",
    "#     if np.amax(abs(np.diff(wave[count]))) > 1.0e-21:\n",
    "#         mark_suddenjump.append(count)\n",
    "# #pathandname='abnormalwaves.pkl'\n",
    "# #fp = open(pathandname,\"w\")\n",
    "# #pickle.dump([mark_nonzeros, mark_suddenjump], fp)\n",
    "# fp.close()        \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The simulated waveforms are sampled with a sampling rate equal to 65535 Hz, \n",
    "# coupled with the longest waveform is ~1.66s, this makes the longest waveform contains 1e5 elements. \n",
    "# Since this code will make other waveforms the same length as the longest length, this requires huge amount of memory,\n",
    "# and makes training very slow and difficult. \n",
    "# Therefore, this codes uses scipy.signal.decimate to down sample the waveforms\n",
    "\n",
    "\n",
    "def padandextractwave(waveformfile, waveformfilekey, waveformfamily, strainkey, wavemaxlength, Dist, no_waves_considered, R, abn_index, alpha):\n",
    "    # Number of simulated waveforms considered\n",
    "    actualwavenumber = no_waves_considered - len(abn_index)\n",
    "    noofwaves = len(waveformfamily[0:actualwavenumber])\n",
    "    \n",
    "    msg = 'Reading waveforms from file and downsampling them by a factor of %s............' %(R)\n",
    "    print(msg)\n",
    "    bar = progressbar.ProgressBar(max_value = actualwavenumber)\n",
    "    \n",
    "    # downsample factor, the downsampled waveform will have length = original length / R\n",
    "    \n",
    "    # Vector used to save the downsampled waveform\n",
    "    downsampled_waveforms = np.array([np.zeros(wavemaxlength / R) for i in range(noofwaves)])\n",
    "    wave_take_in = 0\n",
    "    for i, whichsimulation in enumerate(waveformfamily[0:no_waves_considered]):\n",
    "        \n",
    "        if i in abn_index:\n",
    "            continue\n",
    "        \n",
    "        # convert the unit of the waveform from strain*distance to strain\n",
    "        wave = np.array(waveformfile[waveformfilekey][whichsimulation][strainkey]) / Dist\n",
    "        wavelength = len(wave)\n",
    "        windows = signal.tukey(wavelength, alpha = alpha)\n",
    "        wave = windows * wave\n",
    "        # Pad the waveform with zero so that it has the same length as the longest waveform, \n",
    "        # or whatever length is set by wavemaxlength\n",
    "        temporary = np.pad(wave, (0, wavemaxlength - wavelength), 'constant', constant_values = 0)\n",
    "        \n",
    "        # down sample\n",
    "        downsampled_waveforms[wave_take_in] = signal.decimate(temporary, R, ftype='iir')\n",
    "        bar.update(wave_take_in + 1)\n",
    "        wave_take_in += 1\n",
    "    return downsampled_waveforms\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  1% (23 of 1758) |                      | Elapsed Time: 0:00:00 ETA:  00:00:00"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reading waveforms from file and downsampling them by a factor of 16............\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 98% (1736 of 1758) |################### | Elapsed Time: 0:00:04 ETA:   0:00:00"
     ]
    }
   ],
   "source": [
    "## # Since the original longest waveform length may not be dividable by the down sample vector, \n",
    "# this is to ensure that the length will be dividable. \n",
    "R = 16\n",
    "findmax = 56839#108512\n",
    "\n",
    "findmax = np.ceil(findmax/float(R)) * R\n",
    "\n",
    "# the assumed observation/simulation duration for every waveform \n",
    "Tobs = findmax / 65535.0\n",
    "#start = time.time()\n",
    "SNewaves = padandextractwave(waveformfile, waveformfilekey, waveformfamily, u'strain*dist(cm)', int(findmax), Dist, no_waves_considered, R, abn_index, 0.5)\n",
    "#elapsed = time.time() - start\n",
    "#print(elapsed)\n",
    "# Using the downsampled waveform to compute the new sampling rate\n",
    "New_sr = (len(SNewaves[0])) / Tobs\n",
    "# the new spacing in time\n",
    "New_dt = 1.0 / New_sr\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ASDtxt(x):\n",
    "    \"\"\"This function reads the following noise curves given a detector name.\"\"\"\n",
    "    return {\n",
    "        'LET':'./ASD/ET_D.txt',\n",
    "        'LCE':'./ASD/CE.txt',\n",
    "        'H1': './ASD/ligoII_NS.txt',\n",
    "        'L1': './ASD/ligoII_NS.txt',\n",
    "        'V1': './ASD/virgoII.txt',\n",
    "        'I2': './ASD/ligoII_NS.txt',\n",
    "        'KAGRA': './ASD/ligoII_NS.txt',\n",
    "        'ET_1': './ASD/ET_D.txt',\n",
    "        'ET_2': './ASD/ET_D.txt',\n",
    "        'ET_3': './ASD/ET_D.txt',\n",
    "        'A2': './ASD/ligoII_NS.txt',\n",
    "        'A2.5': './ASD/ligoII_NS.txt',\n",
    "    }[x]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def readnos(detector, f_points):\n",
    "    \"\"\"This function interpolates the noise given the frequency samples.\"\"\"\n",
    "    nos_file = ASDtxt(detector)\n",
    "    f_str = []\n",
    "    ASD_str = []\n",
    "    file = open(nos_file, 'r')\n",
    "    readFile = file.readlines()\n",
    "    file.close()\n",
    "    f = []\n",
    "    ASD = []\n",
    "    \n",
    "    for line in readFile:\n",
    "        p = line.split()\n",
    "        f_str.append(float(p[0]))\n",
    "        ASD_str.append(float(p[1]))\n",
    "    f = np.log10(np.array(f_str))\n",
    "    ASD = np.log10(np.array(ASD_str))\n",
    "    nosinterpolate = interpolate.splrep(f, ASD, w=1.0*np.ones(len(ASD)), s=0)\n",
    "    \n",
    "    nos = interpolate.splev(np.log10(f_points), nosinterpolate, der = 0, ext = 3)\n",
    "    nos = 10**nos\n",
    "    \n",
    "    return nos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def noisegenerator(Tobs, det, SR, df, dt):\n",
    "    \"\"\"This function generates noise based on amplitude spectral density\"\"\"\n",
    "    \n",
    "    # The number of time stamps\n",
    "    Ns = Tobs * SR \n",
    "    \n",
    "    # The number of the frequency samples\n",
    "    Nf = int(Ns // 2 + 1)\n",
    "    \n",
    "    # The frequency sample\n",
    "    fs = np.arange(Nf) * df\n",
    "    \n",
    "    # read ASD\n",
    "    ASD = readnos(det, fs)\n",
    "    #plt.loglog(fs, ASD)\n",
    "    #plt.show()\n",
    "    #dd\n",
    "    \n",
    "    PSD = ASD ** 2\n",
    "    # scale the ASD by the observation time, and this will be the highest amplitude of the generated noise\n",
    "    Amp = np.sqrt(0.25 * Tobs * PSD)\n",
    "    \n",
    "    \n",
    "    idx = np.argwhere(PSD==0.0)\n",
    "    Amp[idx] = 0.0\n",
    "    \n",
    "    real_nos = Amp * np.random.normal(0.0, 1.0, Nf)\n",
    "    img_nos = Amp * np.random.normal(0.0, 1.0, Nf)\n",
    "    \n",
    "    # This is to ensure there is no strange behaviour from noise at low frequency.\n",
    "    # This is because the interpolation function will interpolate strange values at frequencies betweem 1 - 10Hz.\n",
    "    #low_cutoff = 20\n",
    "    #high_cutoff = 2048\n",
    "    \n",
    "    #idx_1 =  int(low_cutoff/df)\n",
    "    #real_nos[0:idx_1] = 0\n",
    "    #img_nos[0:idx_1] = 0\n",
    "    #idx_2 = int(high_cutoff/df)\n",
    "    #real_nos[idx_2:] = 0\n",
    "    #img_nos[idx_2:] = 0\n",
    "    \n",
    "    nos = real_nos + 1j * img_nos\n",
    "\n",
    "    \n",
    "    # Fourier transiform converts the generated noise to the tme domain\n",
    "    fftinput = pyfftw.empty_aligned(len(nos), dtype='complex128')\n",
    "    \n",
    "    fft_object = pyfftw.builders.irfft(fftinput)\n",
    "\n",
    "    nos_realization = Ns* fft_object(nos) * df\n",
    "\n",
    "    return ASD, nos_realization, fs\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def SNR_calculator(waves_in_time_domain, dt, Det):\n",
    "    \n",
    "    length = len(waves_in_time_domain)\n",
    "    \n",
    "    df = 1.0 / (length * dt)\n",
    "    \n",
    "    Nf = int((length // 2 + 1))\n",
    "    \n",
    "    fftinput_for_snr = pyfftw.empty_aligned(length, dtype='complex128')     \n",
    "    fft_object_for_snr = pyfftw.builders.rfft(fftinput_for_snr)      \n",
    "     \n",
    "    # frequency samples\n",
    "    fs = np.arange(Nf) * df\n",
    "    \n",
    "    # Amplitude spectral density\n",
    "    ASD = readnos(Det, fs)\n",
    "        \n",
    "    temporary_wave_in_f = fft_object_for_snr(waves_in_time_domain) * dt\n",
    "    \n",
    "    snr = np.sqrt( 4.0 * sum( abs(temporary_wave_in_f) ** 2 / ASD ** 2 ) * df )\n",
    "    \n",
    "    return snr\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def rescale_to_set_SNR(preset_SNR, SNewaves, dt, Det):\n",
    "    \n",
    "    df = 1.0 / (len(SNewaves[0]) * dt)\n",
    "    fftinput_for_snr = pyfftw.empty_aligned(len(SNewaves[0]), dtype='complex128')     \n",
    "    fft_object_for_snr = pyfftw.builders.rfft(fftinput_for_snr)      \n",
    "    \n",
    "    Nf = int((len(SNewaves[0]) // 2 + 1))\n",
    "    \n",
    "    # frequency samples\n",
    "    fs = np.arange(Nf) * df\n",
    "    \n",
    "    # Amplitude spectral density\n",
    "    ASD = readnos(Det, fs)\n",
    "    msg = 'Rescaling the amplitude of the waveforms so that their optimal SNR is %s.........' %(preset_SNR)\n",
    "    print(msg)\n",
    "    print(\" \")\n",
    "    bar = progressbar.ProgressBar(max_value = len(SNewaves))\n",
    "    \n",
    "    \n",
    "    scaled_snewaves = np.array([np.zeros(len(SNewaves[0])) for i in range(len(SNewaves))])\n",
    "    for i, wave in enumerate(SNewaves):\n",
    "        temporary_wave_in_f = fft_object_for_snr(wave) * dt #np.fft.rfft(wave)*dt #\n",
    "        temporary_snr = np.sqrt( 4.0 * sum( abs(temporary_wave_in_f) ** 2 / ASD ** 2 ) * df )\n",
    "        SNR_factor = preset_SNR / temporary_snr\n",
    "        \n",
    "        scaled_snewaves[i] = SNR_factor * wave\n",
    "        #print(temporary_snr)\n",
    "        #print(  np.sqrt(4.0 * sum(abs(fft_object_for_snr(SNewaves[i]) * dt) **2 / ASD ** 2) * df))\n",
    "        bar.update(i)\n",
    "    \n",
    "    return scaled_snewaves\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def data_generator(seed, ts, dt, Sr, percentage, Det, SNewaves, N_rz, multiplication):\n",
    "    \"\"\"This function generates the data for training/validation/testing.\"\"\"\n",
    "    \n",
    "    np.random.seed(seed)\n",
    "    \n",
    "    # The number of sample will be equal to the number of N_rz(noise realizations)\n",
    "    data = np.array([np.zeros_like(ts) for i in range(N_rz)])\n",
    "    \n",
    "    # Signal to noise ratio\n",
    "    #SNR = np.zeros(N_rz)\n",
    "    \n",
    "    # Number of time stamps\n",
    "    Ns = len(ts)\n",
    "    \n",
    "    # Number of frequency samples\n",
    "    Nf = int(Ns //2 + 1)\n",
    "    \n",
    "    # Observation time\n",
    "    Tobs = ts[-1] + dt\n",
    "    \n",
    "    # spacing in the frequency domain\n",
    "    df = 1.0/Tobs\n",
    "    # frequency samples\n",
    "    fs = np.arange(Nf) * df\n",
    "    \n",
    "    # Amplitude spectral density\n",
    "    ASD = readnos(Det, fs)\n",
    "    \n",
    "    \n",
    "    toolbar_width = N_rz\n",
    "\n",
    "    \n",
    "    \n",
    "    msg = 'Generating noise realizations.......'\n",
    "    print(msg)\n",
    "    print(\" \")\n",
    "    # setup toolbar\n",
    "    bar = progressbar.ProgressBar(max_value=toolbar_width)\n",
    "    \n",
    "\n",
    "    \n",
    "    # Generate noise\n",
    "    for i in range(N_rz):\n",
    "        #if (i+1) % 1000 == 0 & i != N_rz - 1:\n",
    "        #   msg = 'The %s th to %s th noise realizations are now being generated.' %(i+1, i+1000)\n",
    "        #    print(msg)\n",
    "        _, data[i], _ = noisegenerator(Tobs, Det, Sr, df, dt)\n",
    "        bar.update(i+1)\n",
    "\n",
    "\n",
    "\n",
    "    msg = 'Adding noise to signals and converting them back to the time domain after whitening them in the frequency domain.....'\n",
    "    print(msg)\n",
    "    print(\" \")\n",
    "    bar_2 = progressbar.ProgressBar(max_value=toolbar_width)\n",
    "    \n",
    "    \n",
    "    if ts[-1] == signal_duration:   \n",
    "\n",
    "        for i in range(multiplication):\n",
    "            for j in range(len(SNewaves)):\n",
    "\n",
    "                count = i * len(SNewaves) + j\n",
    "                #if (count + 1) % 1000 == 0 and count < 4999:\n",
    "                #    msg = 'The %s th to %s th samples of the data set are now being generated.' %(count + 1,count + 1000)\n",
    "                #    print(msg)\n",
    "                data[count] += SNewaves[j]\n",
    "\n",
    "\n",
    "                fftinput_1 = pyfftw.empty_aligned(len(data[count]), dtype='complex128')\n",
    "                fft_object_1 = pyfftw.builders.rfft(fftinput_1)\n",
    "                temporary = fft_object_1(data[count]) * 1.0/Sr\n",
    "                temporary = temporary / ASD \n",
    "\n",
    "\n",
    "                #SNR[count] = np.sqrt(4.0 * sum(abs(temporary[int(100/df): int(500/df)]) ** 2 * df))\n",
    "                #SNR_factor = SNR_set / SNR[count]\n",
    "                #temporary = temporary * SNR_factor\n",
    "                #if SNR_factor > 1:\n",
    "                #    print(SNR_factor,count) \n",
    "                #print(SNR_factor, np.sqrt(4.0 * sum(abs(temporary) ** 2 * df)))\n",
    "                fftinput_2 = pyfftw.empty_aligned(len(temporary), dtype='complex128')\n",
    "                fft_object_2 = pyfftw.builders.irfft(fftinput_2)\n",
    "                data[count] = Ns * fft_object_2(temporary) * df * np.sqrt(2.0/ Sr)\n",
    "                bar_2.update( count + 1)\n",
    "    elif ts[-1] > signal_duration:\n",
    "        for i in range(multiplication):\n",
    "            for j in range(len(SNewaves)):\n",
    "\n",
    "                count = i * len(SNewaves) + j\n",
    "                #if (count + 1) % 1000 == 0 and count < 4999:\n",
    "                #    msg = 'The %s th to %s th samples of the data set are now being generated.' %(count + 1,count + 1000)\n",
    "                #    print(msg)\n",
    "                # This is to draw a random and determine     \n",
    "                random_shift_percentage = np.random.uniform(-percentage, percentage)\n",
    "                original_starting_point = sample_length / 2 - signal_length / 2\n",
    "                shifted_starting_point = int(original_starting_point + ts[-1] * Sr * random_shift_percentage)\n",
    "                \n",
    "                data[count][shifted_starting_point: shifted_starting_point + signal_length] = data[count][shifted_starting_point: shifted_starting_point + signal_length] + SNewaves[j]\n",
    "        \n",
    "                fftinput_1 = pyfftw.empty_aligned(len(data[count]), dtype='complex128')\n",
    "                fft_object_1 = pyfftw.builders.rfft(fftinput_1)\n",
    "                temporary = fft_object_1(data[count]) * 1.0 / Sr\n",
    "                temporary = temporary / ASD \n",
    "\n",
    "\n",
    "                #SNR[count] = np.sqrt(4.0 * sum(abs(temporary[int(100/df): int(500/df)]) ** 2 * df))\n",
    "                #SNR_factor = SNR_set / SNR[count]\n",
    "                #temporary[int(100/df): int(500/df)] = temporary[int(100/df): int(500/df)] * SNR_factor\n",
    "                \n",
    "                #if SNR_factor > 1:\n",
    "                #    print(SNR_factor,count) \n",
    "                #print(SNR_factor, np.sqrt(4.0 * sum(abs(temporary) ** 2 * df)))\n",
    "                \n",
    "                fftinput_2 = pyfftw.empty_aligned(len(temporary), dtype='complex128')\n",
    "                fft_object_2 = pyfftw.builders.irfft(fftinput_2)\n",
    "                data[count] = Ns * fft_object_2(temporary) * df * np.sqrt(2.0/ Sr)\n",
    "                bar_2.update( count + 1 )\n",
    "    else:\n",
    "        raise Exception('The sample length should be longer than or equal to the signal length') \n",
    "\n",
    "            \n",
    "    for i in range(multiplication * len(SNewaves), N_rz):\n",
    "        fftinput_1 = pyfftw.empty_aligned(len(data[i]), dtype='complex128')\n",
    "        fft_object_1 = pyfftw.builders.rfft(fftinput_1)\n",
    "        temporary = fft_object_1(data[i]) *  1.0 / Sr \n",
    "        temporary = temporary / ASD \n",
    "        \n",
    "        fftinput_2 = pyfftw.empty_aligned(len(temporary), dtype='complex128')\n",
    "        fft_object_2 = pyfftw.builders.irfft(fftinput_2)\n",
    "        data[i] = Ns * fft_object_2(temporary) * df * np.sqrt(2.0/ Sr)\n",
    "        bar_2.update(i + 1)\n",
    "            \n",
    "            \n",
    "    return data #SNR\n",
    "        \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def whiten_data(not_whitened_data_in_time_domain, ASD, dt, SR):\n",
    "    \n",
    "    num = len(not_whitened_data_in_time_domain)\n",
    "    signal_len = len(not_whitened_data_in_time_domain[0])\n",
    "    \n",
    "    whitened_data = np.array([np.zeros(signal_len ) for i in range(num)])\n",
    "    fftinput_in_td = pyfftw.empty_aligned(signal_len, dtype='complex128')\n",
    "    fft_object_to_f = pyfftw.builders.rfft(fftinput_in_td)\n",
    "    \n",
    "    \n",
    "    fftinput_in_fd = pyfftw.empty_aligned(signal_len//2 + 1, dtype='complex128')\n",
    "    fft_object_to_t = pyfftw.builders.irfft(fftinput_in_fd)\n",
    "    \n",
    "    for i, nwd in enumerate(not_whitened_data_in_time_domain):\n",
    "        temp = fft_object_to_f(nwd) * dt / ASD\n",
    "        whitened_data[i] = fft_object_to_t(temp) * np.sqrt(2.0/ SR) / dt\n",
    "        \n",
    "    \n",
    "    return whitened_data \n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# asd, _, _ = noisegenerator(ts[-1] + New_dt , 'H1', New_sr, 1.0/signal_duration, New_dt)\n",
    "# whitened_data = whiten_data(SNewaves, asd, New_dt, New_sr)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# the time stamps \n",
    "signal_length = len(SNewaves[0])\n",
    "signal_duration = (signal_length - 1) * New_dt\n",
    "\n",
    "# applying pad to make the sample longer. This is for the purpose of shifting the signal, so that the signal will appear to be in the centre +- user customised percentage\n",
    "# If no padding is to be applied\n",
    "sample_length = int(signal_length * 1.7 + 2)\n",
    "\n",
    "# time stamps after pad\n",
    "ts = np.arange(sample_length) * New_dt\n",
    "sample_duration = ts[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def shuffle_data(sample, label,  shuffle_times, seed):\n",
    "    np.random.seed(seed)\n",
    "    for i in range(shuffle_times):\n",
    "        state = np.random.randint(0,100)\n",
    "        sample, label = shuffle(sample, label, random_state=state)\n",
    "        \n",
    "    return sample, label\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_model(input_shape, num_classes):\n",
    "    model = Sequential()    # define the type of keras model\n",
    "\n",
    "    # add the layers\n",
    "    # conv1\n",
    "    model.add(Conv2D(8, (1,32), activation='elu', input_shape=input_shape, padding='same'))\n",
    "    # maxpool2\n",
    "    model.add(MaxPool2D((1,8)))\n",
    "    \n",
    "    model.add(Conv2D(8, (1,8), activation='elu', padding='same'))\n",
    "\n",
    "    model.add(MaxPool2D((1,8)))\n",
    "    # conv2\n",
    "    model.add(Conv2D(6, (1,4), activation='elu', padding='same'))\n",
    "    # maxpool2\n",
    "    model.add(MaxPool2D((1,4)))\n",
    "    \n",
    "    model.add(Conv2D(4, (1,4), activation='elu', padding='same'))\n",
    "    # maxpool2\n",
    "    model.add(MaxPool2D((1,2)))\n",
    "    # the input the fully connected layer must be 1-D vector\n",
    "    model.add(Flatten())\n",
    "    model.add(Dense(32, activation='elu'))\n",
    "    model.add(Dropout(0.5))\n",
    "    #dol = keras.layers.Dropout(0.5, noise_shape=None, seed=10)\n",
    "\n",
    "    #model.add(dol)\n",
    "    # add the output layer with softmax actiavtion for classication\n",
    "    model.add(Dense(num_classes, activation='softmax'))\n",
    "    model.summary()\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def kfold_for_diff_SNRs(degree_of_repeat_for_signal, original_waveforms, shift_percentage, data_generator_seed, Det, N_rz, ts, dt, sr,\n",
    "                       batch_size, epochs, kfold_splits, weight_file_name, validation, SNRs):\n",
    "    \n",
    "    if 'data' in globals():\n",
    "        del data, label\n",
    "    \n",
    "    \n",
    "    \n",
    "    presence = len(original_waveforms) * degree_of_repeat_for_signal  #number of samples that contain noise + signal\n",
    "    data_shuffle_seed = data_generator_seed + 10    \n",
    "    kfold_seed = data_generator_seed + 20  # seed for kfold\n",
    "    tscores = [[] for i in range(len(SNRs))]\n",
    "    history_saver = [[] for i in range(len(SNRs))]\n",
    "    num_classes = 2\n",
    "    counter = 0\n",
    "    test_label_saver_for_ROC = [[] for i in range(len(SNRs))]\n",
    "    signal_preds = [[] for i in range(len(SNRs))]\n",
    "    for SNR_set in SNRs:\n",
    "        \n",
    "        weight_file_name_2 = \"SNR_%s.hdf5\" %(SNR_set)\n",
    "        weight_file_name_3 = ''.join([weight_file_name, weight_file_name_2])\n",
    "        \n",
    "        scaled_waveforms = rescale_to_set_SNR(SNR_set, original_waveforms, dt, Det) \n",
    "        # Number of noise realization. This will be the final number of data samples for training + validation + testing\n",
    "        # waveform No. 193 is problematic\n",
    "        data = data_generator(data_generator_seed, ts, dt, sr, shift_percentage, Det, scaled_waveforms, N_rz, degree_of_repeat_for_signal)\n",
    "        label = np.concatenate((np.ones(presence), np.zeros(N_rz - presence)))\n",
    "    \n",
    "        data, label = shuffle_data(data, label,  1, data_shuffle_seed)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "        \"\"\"This part is dedicated to testing the performance of a network by carrying out a k-fold cross validation\"\"\"\n",
    "\n",
    "        # number of time series per batch\n",
    "        # number of full passes of the dataset\n",
    "        # directory to store results in\n",
    "\n",
    "\n",
    "        kfold = StratifiedKFold(kfold_splits, shuffle = True, random_state = kfold_seed)\n",
    "        keras.backend.set_image_data_format('channels_first')\n",
    "        \n",
    "        modelCheck = ModelCheckpoint(weight_file_name_3, monitor='val_acc', verbose=0, save_best_only=True, save_weights_only=True, mode='auto', period=0)\n",
    "\n",
    "\n",
    "        index_for_signal = np.array([i for i in range(presence - validation, presence)])\n",
    "        rest_for_signal = np.array([i for i in range(presence - validation)])\n",
    "        index_for_noise = np.array([i for i in range(N_rz - validation, N_rz)])\n",
    "        rest_for_noise = np.array([i for i in range(presence, N_rz - validation)])\n",
    "\n",
    "        save_for_val = np.concatenate([index_for_signal, index_for_noise])\n",
    "        rest = np.concatenate([rest_for_signal, rest_for_noise])\n",
    "        data_for_val = data[save_for_val]\n",
    "        label_for_val = label[save_for_val]\n",
    "\n",
    "        sample_length = len(data[0])\n",
    "        data_for_val = data_for_val.reshape(-1, 1, 1, sample_length)\n",
    "        label_for_val = keras.utils.to_categorical(label_for_val, num_classes)\n",
    "        \n",
    "        \n",
    "        \n",
    "        msg = \"Training the network on signals with SNR = %s\" %(SNR_set)\n",
    "        print(msg)        \n",
    "        counter_train = 0\n",
    "        for train, test in kfold.split(data[rest],label[rest]):\n",
    "            msg = ''.join([\"Training for the %s\" %(counter_train + 1), \" th times.\"])\n",
    "            print(msg)\n",
    "\n",
    "            data_for_train = data[rest][train]\n",
    "            label_for_train = label[rest][train]\n",
    "\n",
    "            data_for_test = data[rest][test]\n",
    "            label_for_test = label[rest][test]\n",
    "\n",
    "            data_for_train = data_for_train.reshape(-1, 1, 1, sample_length)\n",
    "            data_for_test = data_for_test.reshape(-1, 1, 1, sample_length)\n",
    "\n",
    "            input_shape = data_for_train.shape[1:]\n",
    "            print(input_shape)\n",
    "            label_for_train = keras.utils.to_categorical(label_for_train , num_classes)\n",
    "            label_for_test = keras.utils.to_categorical(label_for_test, num_classes)\n",
    "\n",
    "            if \"model\" in locals() or \"model\" in globals():\n",
    "                del model\n",
    "            model = make_model(input_shape, num_classes)\n",
    "\n",
    "            # compile the model #adam = keras.optimizers.Adam(lr=0.01)\n",
    "            model.compile(loss='categorical_crossentropy', optimizer= Nadam(), metrics=['accuracy'])\n",
    "\n",
    "            history = model.fit(data_for_train, label_for_train, batch_size=batch_size, epochs=epochs, \n",
    "                                verbose=1, validation_data=(data_for_val, label_for_val), callbacks = [modelCheck], shuffle = False)\n",
    "\n",
    "\n",
    "            model.load_weights(weight_file_name_3)\n",
    "            # evaluate\n",
    "            eval_results = model.evaluate(data_for_test, label_for_test, verbose=1)\n",
    "            print('The result of testing the model against test data is:')\n",
    "            print('Test loss: %s'%(eval_results[0]))\n",
    "            print('Test accuracy %s:' %(eval_results[1]))\n",
    "            print(' ')\n",
    "            tscores[counter].append(eval_results)\n",
    "            history_saver[counter].append(history)\n",
    "            signal_preds[counter].append(model.predict(data_for_test))\n",
    "            test_label_saver_for_ROC[counter].append(label_for_test) \n",
    "            \n",
    "            \n",
    "            counter_train += 1\n",
    "        counter += 1\n",
    "    return tscores, history_saver, signal_preds, test_label_saver_for_ROC\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 35% (624 of 1758) |#######              | Elapsed Time: 0:00:00 ETA:   0:00:00"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rescaling the amplitude of the waveforms so that their optimal SNR is 5.0.........\n",
      " \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% (23 of 50000) |                     | Elapsed Time: 0:00:00 ETA:   0:03:43"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generating noise realizations.......\n",
      " \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0% (139 of 50000) |                    | Elapsed Time: 0:00:00 ETA:   0:00:35"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Adding noise to signals and converting them back to the time domain after whitening them in the frequency domain.....\n",
      " \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% (49887 of 50000) |################# | Elapsed Time: 0:00:35 ETA:   0:00:00"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training the network on signals with SNR = 5.0\n",
      "Training for the 1 th times.\n",
      "(1, 1, 6042)\n"
     ]
    },
    {
     "ename": "NameError",
     "evalue": "global name 'dol' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-21-34e386c26274>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     16\u001b[0m \u001b[0mSNRs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;36m5.0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m tscores, history, signal_preds, test_label_saver_for_ROC = kfold_for_diff_SNRs(degree_of_repeat_for_signal, SNewaves, shift_percentage, data_generator_seed, Det, N_rz, ts, New_dt, New_sr,\n\u001b[0;32m---> 18\u001b[0;31m                                                                                batch_size, epochs, kfold_splits, weight_file_name, validation, SNRs)\n\u001b[0m",
      "\u001b[0;32m<ipython-input-20-61a5876ccd2e>\u001b[0m in \u001b[0;36mkfold_for_diff_SNRs\u001b[0;34m(degree_of_repeat_for_signal, original_waveforms, shift_percentage, data_generator_seed, Det, N_rz, ts, dt, sr, batch_size, epochs, kfold_splits, weight_file_name, validation, SNRs)\u001b[0m\n\u001b[1;32m     84\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0;34m\"model\"\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mlocals\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m\"model\"\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mglobals\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     85\u001b[0m                 \u001b[0;32mdel\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 86\u001b[0;31m             \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmake_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_shape\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnum_classes\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     87\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     88\u001b[0m             \u001b[0;31m# compile the model #adam = keras.optimizers.Adam(lr=0.01)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-19-dbd9cf242916>\u001b[0m in \u001b[0;36mmake_model\u001b[0;34m(input_shape, num_classes)\u001b[0m\n\u001b[1;32m     25\u001b[0m     \u001b[0;31m#dol = keras.layers.Dropout(0.5, noise_shape=None, seed=10)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     26\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 27\u001b[0;31m     \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdol\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     28\u001b[0m     \u001b[0;31m# add the output layer with softmax actiavtion for classication\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     29\u001b[0m     \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mDense\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnum_classes\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mactivation\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'softmax'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: global name 'dol' is not defined"
     ]
    }
   ],
   "source": [
    "degree_of_repeat_for_signal = 14\n",
    "shift_percentage = 0.2\n",
    "data_generator_seed = 10\n",
    "Det = 'H1'\n",
    "N_rz = 50000\n",
    "\n",
    "batch_size = 64\n",
    "epochs =30\n",
    "kfold_splits= 10\n",
    "weight_file_name = 'results/testing_weights_at_'\n",
    "validation = 1000\n",
    "#SNRs = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0]\n",
    "#SNRs = [1.0, 2.0, 3.0, 4.0]\n",
    "#SNRs = [5.0, 6.0, 7.0, 8.0]\n",
    "#SNRs = [9.0,10.0]\n",
    "SNRs = [5.0]\n",
    "tscores, history, signal_preds, test_label_saver_for_ROC = kfold_for_diff_SNRs(degree_of_repeat_for_signal, SNewaves, shift_percentage, data_generator_seed, Det, N_rz, ts, New_dt, New_sr,\n",
    "                                                                               batch_size, epochs, kfold_splits, weight_file_name, validation, SNRs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The mean of test loss is 0.4948273861971872\n",
      " \n",
      "The standard deviation of test loss is 0.015794685550233757\n",
      " \n",
      "The mean of test accuracy is 0.7604174784527127\n",
      " \n",
      "The standard deviation of test accuracy is 0.010653911114099877\n"
     ]
    }
   ],
   "source": [
    "t_scores = np.array([np.zeros((10, 2)) for i in range(len(tscores))])\n",
    "for i in range(len(tscores)):\n",
    "    for j in range(10):\n",
    "        t_scores[i][j][0] = tscores[i][j][0]\n",
    "        t_scores[i][j][1] = tscores[i][j][1]\n",
    "\n",
    "msg = 'The mean of test loss is %s' %(np.mean(t_scores[0][:,0]))\n",
    "print(msg)\n",
    "print(' ')\n",
    "\n",
    "msg = 'The standard deviation of test loss is %s' %(np.std(t_scores[0][:,0]))\n",
    "print(msg)\n",
    "print(' ')\n",
    "\n",
    "msg = 'The mean of test accuracy is %s' %(np.mean(t_scores[0][:,1]))\n",
    "print(msg)\n",
    "print(' ')\n",
    "\n",
    "msg = 'The standard deviation of test accuracy is %s' %(np.std(t_scores[0][:,1]))\n",
    "print(msg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plotlossacc(history, fontsize, epochs):\n",
    "    fig , axs = plt.subplots(2,1, sharex = True)\n",
    "    axs = axs.ravel()\n",
    "    # plot history\n",
    "    counter = 0\n",
    "    for history_i in history:\n",
    "        if counter == 0:\n",
    "            axs[0].plot(np.arange(epochs) + 1, history_i.history['loss'], label = 'Loss', linewidth = 1, color = 'b')\n",
    "            axs[0].plot(np.arange(epochs) + 1, history_i.history['val_loss'], label = 'Validation Loss', linewidth = 1, color = 'r')\n",
    "\n",
    "            axs[1].plot(np.arange(epochs) + 1, history_i.history['acc'], label = 'Accuracy', linewidth = 1, color =  'b')\n",
    "            axs[1].plot(np.arange(epochs) + 1, history_i.history['val_acc'], label = 'Validation Accurarcy', linewidth = 1, color =  'r')\n",
    "            # set labels\n",
    "            axs[0].set_ylabel('Loss', fontsize = fontsize)\n",
    "            axs[1].set_xlabel('Epoch', fontsize = fontsize)\n",
    "            axs[1].set_ylabel('Acc', fontsize = fontsize)\n",
    "        \n",
    "        \n",
    "        \n",
    "            # legends\n",
    "            axs[0].legend(fontsize = fontsize)\n",
    "            axs[1].legend(fontsize = fontsize)\n",
    "        else:\n",
    "            axs[0].plot(np.arange(epochs) + 1, history_i.history['loss'], linewidth = 1, color = 'b')\n",
    "            axs[0].plot(np.arange(epochs) + 1, history_i.history['val_loss'], linewidth = 1, color = 'r')\n",
    "\n",
    "            axs[1].plot(np.arange(epochs) + 1, history_i.history['acc'], linewidth = 1, color = 'b')\n",
    "            axs[1].plot(np.arange(epochs) + 1, history_i.history['val_acc'], linewidth = 1, color = 'r')\n",
    "            # set labels\n",
    "            axs[0].set_ylabel('Loss', fontsize = fontsize)\n",
    "            axs[1].set_xlabel('Epoch', fontsize = fontsize)\n",
    "            axs[1].set_ylabel('Acc', fontsize = fontsize)\n",
    "        \n",
    "        \n",
    "        \n",
    "            # legends\n",
    "            axs[0].legend(fontsize = fontsize)\n",
    "            axs[1].legend(fontsize = fontsize)\n",
    "        counter += 1\n",
    "    # grids\n",
    "    axs[0].grid()\n",
    "    axs[1].grid()\n",
    "    axs[0].set_xlim([1, epochs])\n",
    "    axs[0].set_ylim(bottom = 0)\n",
    "\n",
    "    axs[1].set_xlim([1, epochs])\n",
    "    axs[1].set_ylim(top = 1)\n",
    "\n",
    "    plt.subplots_adjust(left = 0.1, bottom = 0.1, right = 0.90, top = 0.95)\n",
    "    for ax in axs:\n",
    "        for tick in ax.xaxis.get_major_ticks():\n",
    "            tick.label1.set_fontsize(fontsize)\n",
    "            tick.label1.set_fontweight('normal')\n",
    "        for tick in ax.yaxis.get_major_ticks():\n",
    "            tick.label1.set_fontsize(fontsize)\n",
    "            tick.label1.set_fontweight('normal')\n",
    "    plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "fontsize = 20\n",
    "#history = history_saver[0][0]\n",
    "plotlossacc(history[0], fontsize, epochs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def color_shelf(SNR):\n",
    "    nf = 255.0\n",
    "    RGB_choice = {\n",
    "        1 : [255.0, 206.0, 0.0],\n",
    "        2 : [49.0, 51.0, 53.0],\n",
    "        3 : [27.0, 161.0, 226.0],\n",
    "        4 : [222.0, 93.0, 90.0],\n",
    "        5 : [128.0, 211.0, 2.0],\n",
    "        6 : [194.0, 45.0, 140.0],\n",
    "        7 : [227.0, 37.0, 107.0],\n",
    "        8 : [255.0, 146.0, 93.0],\n",
    "        9 : [33.0, 206.0, 142.0],\n",
    "        10: [70.0, 33.0, 180.0]\n",
    "    }\n",
    "    #RGB[SNR]\n",
    "    RGB = [x /nf for x in RGB_choice[SNR]]\n",
    "    return RGB\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plotlossacc_together(history, fontsize, SNR, Loss_top, Loss_bottom, Acc_top, Acc_bottom):\n",
    "    fig , axs = plt.subplots(2,1, sharex = True)\n",
    "    axs = axs.ravel()\n",
    "    # plot history\n",
    "    for history_10 in history:\n",
    "        counter = 0\n",
    "        for history_i in history_10[0:1]:\n",
    "            if counter == 0:\n",
    "                axs[0].plot(np.arange(epochs) + 1, history_i.history['loss'], label = 'Loss at SNR = %s' %(SNR), linewidth = 1, linestyle ='dashdot', color = color_shelf(SNR))\n",
    "                axs[0].plot(np.arange(epochs) + 1, history_i.history['val_loss'], label = 'Validation Loss at SNR = %s' %(SNR), linewidth = 1, linestyle ='-', color = color_shelf(SNR))\n",
    "\n",
    "                axs[1].plot(np.arange(epochs) + 1, history_i.history['acc'], label = 'Accuracy at SNR = %s' %(SNR), linewidth = 1,  linestyle ='dashdot', color =  color_shelf(SNR))\n",
    "                axs[1].plot(np.arange(epochs) + 1, history_i.history['val_acc'], label = 'Validation Accurarcy at SNR = %s' %(SNR), linewidth = 1, linestyle ='-', color =  color_shelf(SNR))\n",
    "                # set labels\n",
    "                axs[0].set_ylabel('Loss', fontsize = fontsize)\n",
    "                axs[1].set_xlabel('Epoch', fontsize = fontsize)\n",
    "                axs[1].set_ylabel('Acc', fontsize = fontsize)\n",
    "\n",
    "\n",
    "\n",
    "                # legends\n",
    "                axs[0].legend(fontsize = fontsize - 10)\n",
    "                axs[1].legend(fontsize = fontsize - 10)\n",
    "            else:\n",
    "                axs[0].plot(np.arange(epochs) + 1, history_i.history['loss'], linewidth = 1, linestyle ='dashdot', color = color_shelf(SNR))\n",
    "                axs[0].plot(np.arange(epochs) + 1, history_i.history['val_loss'], linewidth = 1, linestyle ='-', color = color_shelf(SNR))\n",
    "\n",
    "                axs[1].plot(np.arange(epochs) + 1, history_i.history['acc'], linewidth = 1, linestyle ='dashdot', color = color_shelf(SNR))\n",
    "                axs[1].plot(np.arange(epochs) + 1, history_i.history['val_acc'], linewidth = 1, linestyle ='-', color = color_shelf(SNR))\n",
    "                # set labels\n",
    "                axs[0].set_ylabel('Loss', fontsize = fontsize)\n",
    "                axs[1].set_xlabel('Epoch', fontsize = fontsize)\n",
    "                axs[1].set_ylabel('Acc', fontsize = fontsize)\n",
    "\n",
    "\n",
    "\n",
    "                # legends\n",
    "                axs[0].legend(fontsize = fontsize - 10)\n",
    "                axs[1].legend(fontsize = fontsize - 10)\n",
    "            counter += 1\n",
    "        SNR +=1    \n",
    "    # grids\n",
    "    axs[0].grid()\n",
    "    axs[1].grid()\n",
    "    axs[0].set_xlim([1, epochs])\n",
    "    #axs[0].set_ylim(top = Loss_top, bottom = Loss_bottom)\n",
    "    axs[0].set_xticks(np.arange(epochs) +1)\n",
    "    axs[1].set_xlim([1, epochs])\n",
    "    #axs[1].set_ylim(top = Acc_top, bottom = Acc_bottom)\n",
    "    axs[1].set_xticks(np.arange(epochs) +1)\n",
    "    plt.subplots_adjust(left = 0.1, bottom = 0.1, right = 0.90, top = 0.95)\n",
    "    for ax in axs:\n",
    "        for tick in ax.xaxis.get_major_ticks():\n",
    "            tick.label1.set_fontsize(fontsize)\n",
    "            tick.label1.set_fontweight('normal')\n",
    "        for tick in ax.yaxis.get_major_ticks():\n",
    "            tick.label1.set_fontsize(fontsize)\n",
    "            tick.label1.set_fontweight('normal')\n",
    "    plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "fontsize = 20\n",
    "#history = history_saver[0][0]\n",
    "Loss_top = 0.8\n",
    "Loss_bottom = 0.5\n",
    "Acc_top = 0.8\n",
    "Acc_bottom = 0.3\n",
    "plotlossacc_together(history, fontsize, 1, Loss_top, Loss_bottom, Acc_top, Acc_bottom)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fontsize = 20\n",
    "#history = history_saver[0][0]\n",
    "Loss_top = 0.8\n",
    "Loss_bottom = 0.3\n",
    "Acc_top = 1.0\n",
    "Acc_bottom = 0.3\n",
    "plotlossacc_together(history[3:7], fontsize, 4, Loss_top, Loss_bottom, Acc_top, Acc_bottom)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [],
   "source": [
    "fontsize = 20\n",
    "#history = history_saver[0][0]\n",
    "Loss_top = 0.6\n",
    "Loss_bottom = 0.0\n",
    "Acc_top = 1.0\n",
    "Acc_bottom = 0.6\n",
    "plotlossacc_together(history[7:], fontsize, 8, Loss_top, Loss_bottom, Acc_top, Acc_bottom)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "pathandname='1736waveformSNR110randombegin.pkl'\n",
    "fp = open(pathandname,\"w\")\n",
    "pickle.dump([tscores, history, signal_preds, test_label_saver_for_ROC], fp)\n",
    "fp.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "#load the data\n",
    "with open(\"1824waveformSNR8910.pkl\") as f:\n",
    "    tscores, history, signal_preds, test_label_saver_for_ROC = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "fontsize =20\n",
    "i = 0\n",
    "SNR = i + 5\n",
    "epochs =20\n",
    "plot_tscores(t_scores[i], fontsize)\n",
    "#plot_roc(test_label_saver_for_ROC[i], signal_preds[i], SNR)\n",
    "#plotlossacc(history[i], fontsize)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_tscores(tscores, fontsize):\n",
    "    ksplit = len(tscores)    \n",
    "    fig = plt.figure(figsize=(6,2.5), dpi= 100, facecolor='w', edgecolor='k')\n",
    "\n",
    "    plt.scatter(np.arange(ksplit ) + 1, tscores[:,1], color = 'r', s = 200)\n",
    "    plt.grid()\n",
    "    ax = plt.gca()\n",
    "    for tick in ax.xaxis.get_major_ticks():\n",
    "        tick.label1.set_fontsize(fontsize)\n",
    "        tick.label1.set_fontweight('normal')\n",
    "    for tick in ax.yaxis.get_major_ticks():\n",
    "        tick.label1.set_fontsize(fontsize)\n",
    "        tick.label1.set_fontweight('normal')\n",
    "    \n",
    "    plt.xlim([1, ksplit])\n",
    "    #plt.ylim([0.94, 1])\n",
    "\n",
    "    #plt.xlabel('K-Fold iteration',fontsize = fontsize)\n",
    "    plt.ylabel('Test accuracy',fontsize = fontsize)\n",
    "\n",
    "    plt.show()\n",
    "    fig = plt.figure(figsize=(6,2.5), dpi= 100, facecolor='w', edgecolor='k')\n",
    "\n",
    "    plt.scatter(np.arange(ksplit ) + 1, tscores[:,0], color = 'r', s = 200)\n",
    "    plt.grid()\n",
    "    ax = plt.gca()\n",
    "    for tick in ax.xaxis.get_major_ticks():\n",
    "        tick.label1.set_fontsize(fontsize)\n",
    "        tick.label1.set_fontweight('normal')\n",
    "    for tick in ax.yaxis.get_major_ticks():\n",
    "        tick.label1.set_fontsize(fontsize)\n",
    "        tick.label1.set_fontweight('normal')\n",
    "    \n",
    "    #plt.xlim([bottom = 0])\n",
    "    #plt.ylim([0.94, 1])\n",
    "\n",
    "    plt.xlabel('K-Fold iteration',fontsize = fontsize)\n",
    "    plt.ylabel('Test loss',fontsize = fontsize)\n",
    "\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_roc(test_label, signal_preds, SNR, plot):\n",
    "    if plot == True:\n",
    "        fig = plt.figure()\n",
    "    fa = []#[[] for i in range(len(test_label))]\n",
    "    ta = []#[[] for i in range(len(test_label))]\n",
    "    for i, j in zip(test_label, signal_preds):\n",
    "        tem_fa, tem_ta, _ = metrics.roc_curve(i[:,1], j[:,1])\n",
    "        fa.append(tem_fa)\n",
    "        ta.append(tem_ta)\n",
    "        if plot == True:\n",
    "            plt.plot(tem_fa, tem_ta, linewidth = 2, color = 'b')\n",
    "            plt.xlabel('False alarm probability',fontsize = fontsize)\n",
    "            plt.ylabel('True alarm probability',fontsize = fontsize)\n",
    "            plt.title('ROC curve for SNR %s'%(SNR), fontsize = fontsize)\n",
    "            plt.xlim([0, 1])\n",
    "            plt.ylim([0, 1])\n",
    "            plt.subplots_adjust(left = 0.1, bottom = 0.1, right = 0.90, top = 0.95)\n",
    "            \n",
    "             \n",
    "    if plot == True:            \n",
    "        plt.grid()\n",
    "        ax = plt.gca()\n",
    "        for tick in ax.xaxis.get_major_ticks():\n",
    "            tick.label1.set_fontsize(fontsize)\n",
    "            tick.label1.set_fontweight('normal')\n",
    "        for tick in ax.yaxis.get_major_ticks():\n",
    "            tick.label1.set_fontsize(fontsize)\n",
    "            tick.label1.set_fontweight('normal')\n",
    "        plt.show()\n",
    "    return fa, ta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "def interpta(fa, ta, given_FAP):\n",
    "     \n",
    "    ta_fun = interpolate.interp1d(fa, ta)\n",
    "    \n",
    "    ta_interp = ta_fun(given_FAP)\n",
    "    \n",
    "    return ta_interp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.57456645, 0.41542672, 0.41542036, 0.41971365, 0.417178  , 0.4235626 , 0.42090472, 0.41190883, 0.40760758, 0.55043136])"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t_scores[0][:,0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "given_FAR = 0.1;\n",
    "given_FAR2 = 0.01;\n",
    "given_FAR3 = 0.001;\n",
    "\n",
    "SNRs = np.array([1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0])\n",
    "ta_interp_01 = np.zeros(len(SNRs))\n",
    "ta_interp_001 = np.zeros(len(SNRs))\n",
    "ta_interp_0001 = np.zeros(len(SNRs))\n",
    "mark_best =[]\n",
    "for i in range(len(SNRs)):  \n",
    "    fa, ta = plot_roc(test_label_saver_for_ROC[i], signal_preds[i], SNRs[i], False)\n",
    "    j = np.argmin(t_scores[i][:,0])     \n",
    "    mark_best.append(j)\n",
    "    ta_interp_01[i] = interpta(fa[j], ta[j], given_FAR)\n",
    "    ta_interp_001[i] = interpta(fa[j], ta[j], given_FAR2)\n",
    "    ta_interp_0001[i] = interpta(fa[j], ta[j], given_FAR3)\n",
    "plt.plot(SNRs, ta_interp_01, linewidth = 2.0, color = 'r', label = 'False Alarm Rate = 0.1')\n",
    "plt.plot(SNRs, ta_interp_001, linewidth = 2.0, color = 'b', label = 'False Alarm Rate = 0.01')\n",
    "plt.plot(SNRs, ta_interp_0001, linewidth = 2.0, color = 'g', label = 'False Alarm Rate = 0.001')\n",
    "plt.grid()\n",
    "ax = plt.gca()\n",
    "plt.xlim([1, 10])\n",
    "plt.ylim([0, 1])\n",
    "plt.legend(loc = 7, fontsize =20)\n",
    "plt.subplots_adjust(left = 0.1, bottom = 0.1, right = 0.90, top = 0.95)\n",
    "\n",
    "for tick in ax.xaxis.get_major_ticks():\n",
    "    tick.label1.set_fontsize(fontsize)\n",
    "    tick.label1.set_fontweight('normal')\n",
    "for tick in ax.yaxis.get_major_ticks():\n",
    "    tick.label1.set_fontsize(fontsize)\n",
    "    tick.label1.set_fontweight('normal')\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1801"
      ]
     },
     "execution_count": 145,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(test_label_saver_for_ROC[0][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(len(SNRs)): \n",
    "    tem_fa, tem_ta, _ = metrics.roc_curve(test_label_saver_for_ROC[i][mark_best[i]][:,1], signal_preds[i][mark_best[i]][:,1])\n",
    "    plt.plot(tem_fa, tem_ta, linewidth = 2, color = color_shelf(SNRs[i]) , label = 'SNR = %s' %(SNRs[i]))\n",
    "    plt.xlabel('False alarm probability',fontsize = fontsize)\n",
    "    plt.ylabel('True alarm probability',fontsize = fontsize)\n",
    "    plt.title('ROC curves for SNR 1-10', fontsize = fontsize)\n",
    "    plt.xlim([0, 1])\n",
    "    plt.ylim([0, 1])\n",
    "    plt.subplots_adjust(left = 0.1, bottom = 0.1, right = 0.90, top = 0.95)\n",
    "plt.grid()\n",
    "plt.legend()\n",
    "ax = plt.gca()\n",
    "for tick in ax.xaxis.get_major_ticks():\n",
    "    tick.label1.set_fontsize(fontsize)\n",
    "    tick.label1.set_fontweight('normal')\n",
    "for tick in ax.yaxis.get_major_ticks():\n",
    "    tick.label1.set_fontsize(fontsize)\n",
    "    tick.label1.set_fontweight('normal')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "ta_interp2 = np.concatenate((ta_interp2, ta_interp))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.plot(np.arange(10)+1, ta_interp2)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(test_label_saver_for_ROC[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "j = np.argmin(t_scores[i][:,0])\n",
    "fa, ta, _ = metrics.roc_curve(test_label_saver_for_ROC[0][j][:,1], signal_preds[0][j][:,1])\n",
    "fig = plt.figure()\n",
    "plt.plot(fa, ta, linewidth = 2, color = 'b')\n",
    "plt.xlabel('False alarm probability',fontsize = fontsize)\n",
    "plt.ylabel('True alarm probability',fontsize = fontsize)\n",
    "plt.title('ROC curve for SNR %s'%(5), fontsize = fontsize)\n",
    "plt.xlim([0, 1])\n",
    "plt.ylim([0, 1])\n",
    "plt.subplots_adjust(left = 0.1, bottom = 0.1, right = 0.90, top = 0.95)\n",
    "\n",
    "plt.grid()\n",
    "ax = plt.gca()\n",
    "ax.set_xticks([0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0])\n",
    "for tick in ax.xaxis.get_major_ticks():\n",
    "    tick.label1.set_fontsize(fontsize)\n",
    "    tick.label1.set_fontweight('normal')\n",
    "for tick in ax.yaxis.get_major_ticks():\n",
    "    tick.label1.set_fontsize(fontsize)\n",
    "    tick.label1.set_fontweight('normal')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "fa, ta, _ = metrics.roc_curve(test_label_saver_for_ROC[0][j][:,1], signal_preds[0][j][:,1])\n",
    "fig = plt.figure()\n",
    "plt.loglog(fa, ta, linewidth = 2, color = 'b')\n",
    "plt.xlabel('False alarm probability',fontsize = fontsize)\n",
    "plt.ylabel('True alarm probability',fontsize = fontsize)\n",
    "plt.title('ROC curve for SNR %s'%(5), fontsize = fontsize)\n",
    "plt.xlim([0.001, 1])\n",
    "plt.ylim([0.001, 1])\n",
    "plt.subplots_adjust(left = 0.1, bottom = 0.1, right = 0.90, top = 0.95)\n",
    "\n",
    "plt.grid()\n",
    "ax = plt.gca()\n",
    "for tick in ax.xaxis.get_major_ticks():\n",
    "    tick.label1.set_fontsize(fontsize)\n",
    "    tick.label1.set_fontweight('normal')\n",
    "for tick in ax.yaxis.get_major_ticks():\n",
    "    tick.label1.set_fontsize(fontsize)\n",
    "    tick.label1.set_fontweight('normal')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}