MLProject.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Imports\n",
    "\n",
    "# pandas\n",
    "import pandas as pd\n",
    "from pandas import Series,DataFrame\n",
    "\n",
    "# numpy, matplotlib, seaborn\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "sns.set_style('whitegrid')\n",
    "%matplotlib inline\n",
    "\n",
    "# machine learning\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.svm import SVC, LinearSVC\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.naive_bayes import GaussianNB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Load data and select features\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/danielwu/anaconda/lib/python3.6/site-packages/IPython/core/interactiveshell.py:2850: DtypeWarning: Columns (471,473,474,487,488,489,490,491,492,493,494,495,496,497,498,499,500,501,502,503,504,505,506,507,508,509,510,511,512,513,514,515,516,517,518,519,520,521,522,523,524,525,526,527,528,529,530,531,532,533,534,535,536,537,538,539,540,541,542,543,544,545,546,547,548,549,550,551,552,553,554,555,556,557,558,559,560,561,562,563,569,570,572,573,574,575,576,577,578,579,580,581,582,583,584,585,586,587,588,589,590,591,592,593,594,595,596,597,599,601,606,607,608,609,610,611,612,613,614,615,616,617,618,619,620,621,624,625,626,627,628,629,630,631,632,633,634,636,637,638,639,640,641,642,643,644,645,646,647,648,649,650,651,652,653,654,655,656,657,658,659,660,661,663,664,665,666,667,668,669,670,671,672,673,674,675,676,677,678,679,680,681,682,683,684,685,686,687,688,689,690,691,692,693,694,695,696,697,698,699,700,701,702,703,704,705,706,707,708,709,710,711,712,713,714,715,716,717,718,719,720,721,722,723,724,725,726,727,728,729,730,731,732,733,734,735,736,737,738,739,745,746,748,749,750,751,752,753,754,755,756,757,758,759,760,761,762,763,764,765,766,767,770,771,776,777,778,779,780,781,782,783,784,785,786,787,788,789,790,791,794,795,797,798,799,800,801,802,803,804,806,807,808,809,810,811,812,813,814,815,816,817,818,819,820,821,822,823,824,825,826,827,828,829,830,831) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  if self.run_code(code, result):\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train SVM for Diagnosis and SVR for ADAS and Ventricles\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[MICE] Completing matrix with shape (12012, 22)\n",
      "[MICE] Starting imputation round 1/110, elapsed time 0.004\n",
      "[MICE] Starting imputation round 2/110, elapsed time 0.130\n",
      "[MICE] Starting imputation round 3/110, elapsed time 0.178\n",
      "[MICE] Starting imputation round 4/110, elapsed time 0.230\n",
      "[MICE] Starting imputation round 5/110, elapsed time 0.278\n",
      "[MICE] Starting imputation round 6/110, elapsed time 0.323\n",
      "[MICE] Starting imputation round 7/110, elapsed time 0.367\n",
      "[MICE] Starting imputation round 8/110, elapsed time 0.412\n",
      "[MICE] Starting imputation round 9/110, elapsed time 0.459\n",
      "[MICE] Starting imputation round 10/110, elapsed time 0.504\n",
      "[MICE] Starting imputation round 11/110, elapsed time 0.549\n",
      "[MICE] Starting imputation round 12/110, elapsed time 0.594\n",
      "[MICE] Starting imputation round 13/110, elapsed time 0.640\n",
      "[MICE] Starting imputation round 14/110, elapsed time 0.689\n",
      "[MICE] Starting imputation round 15/110, elapsed time 0.735\n",
      "[MICE] Starting imputation round 16/110, elapsed time 0.780\n",
      "[MICE] Starting imputation round 17/110, elapsed time 0.826\n",
      "[MICE] Starting imputation round 18/110, elapsed time 0.872\n",
      "[MICE] Starting imputation round 19/110, elapsed time 0.919\n",
      "[MICE] Starting imputation round 20/110, elapsed time 0.966\n",
      "[MICE] Starting imputation round 21/110, elapsed time 1.012\n",
      "[MICE] Starting imputation round 22/110, elapsed time 1.059\n",
      "[MICE] Starting imputation round 23/110, elapsed time 1.104\n",
      "[MICE] Starting imputation round 24/110, elapsed time 1.153\n",
      "[MICE] Starting imputation round 25/110, elapsed time 1.199\n",
      "[MICE] Starting imputation round 26/110, elapsed time 1.245\n",
      "[MICE] Starting imputation round 27/110, elapsed time 1.292\n",
      "[MICE] Starting imputation round 28/110, elapsed time 1.339\n",
      "[MICE] Starting imputation round 29/110, elapsed time 1.388\n",
      "[MICE] Starting imputation round 30/110, elapsed time 1.439\n",
      "[MICE] Starting imputation round 31/110, elapsed time 1.486\n",
      "[MICE] Starting imputation round 32/110, elapsed time 1.533\n",
      "[MICE] Starting imputation round 33/110, elapsed time 1.581\n",
      "[MICE] Starting imputation round 34/110, elapsed time 1.631\n",
      "[MICE] Starting imputation round 35/110, elapsed time 1.679\n",
      "[MICE] Starting imputation round 36/110, elapsed time 1.727\n",
      "[MICE] Starting imputation round 37/110, elapsed time 1.774\n",
      "[MICE] Starting imputation round 38/110, elapsed time 1.820\n",
      "[MICE] Starting imputation round 39/110, elapsed time 1.868\n",
      "[MICE] Starting imputation round 40/110, elapsed time 1.916\n",
      "[MICE] Starting imputation round 41/110, elapsed time 1.964\n",
      "[MICE] Starting imputation round 42/110, elapsed time 2.012\n",
      "[MICE] Starting imputation round 43/110, elapsed time 2.058\n",
      "[MICE] Starting imputation round 44/110, elapsed time 2.107\n",
      "[MICE] Starting imputation round 45/110, elapsed time 2.155\n",
      "[MICE] Starting imputation round 46/110, elapsed time 2.203\n",
      "[MICE] Starting imputation round 47/110, elapsed time 2.251\n",
      "[MICE] Starting imputation round 48/110, elapsed time 2.297\n",
      "[MICE] Starting imputation round 49/110, elapsed time 2.346\n",
      "[MICE] Starting imputation round 50/110, elapsed time 2.392\n",
      "[MICE] Starting imputation round 51/110, elapsed time 2.439\n",
      "[MICE] Starting imputation round 52/110, elapsed time 2.485\n",
      "[MICE] Starting imputation round 53/110, elapsed time 2.544\n",
      "[MICE] Starting imputation round 54/110, elapsed time 2.607\n",
      "[MICE] Starting imputation round 55/110, elapsed time 2.657\n",
      "[MICE] Starting imputation round 56/110, elapsed time 2.708\n",
      "[MICE] Starting imputation round 57/110, elapsed time 2.757\n",
      "[MICE] Starting imputation round 58/110, elapsed time 2.807\n",
      "[MICE] Starting imputation round 59/110, elapsed time 2.858\n",
      "[MICE] Starting imputation round 60/110, elapsed time 2.905\n",
      "[MICE] Starting imputation round 61/110, elapsed time 2.958\n",
      "[MICE] Starting imputation round 62/110, elapsed time 3.015\n",
      "[MICE] Starting imputation round 63/110, elapsed time 3.063\n",
      "[MICE] Starting imputation round 64/110, elapsed time 3.114\n",
      "[MICE] Starting imputation round 65/110, elapsed time 3.162\n",
      "[MICE] Starting imputation round 66/110, elapsed time 3.226\n",
      "[MICE] Starting imputation round 67/110, elapsed time 3.299\n",
      "[MICE] Starting imputation round 68/110, elapsed time 3.367\n",
      "[MICE] Starting imputation round 69/110, elapsed time 3.434\n",
      "[MICE] Starting imputation round 70/110, elapsed time 3.496\n",
      "[MICE] Starting imputation round 71/110, elapsed time 3.547\n",
      "[MICE] Starting imputation round 72/110, elapsed time 3.595\n",
      "[MICE] Starting imputation round 73/110, elapsed time 3.645\n",
      "[MICE] Starting imputation round 74/110, elapsed time 3.695\n",
      "[MICE] Starting imputation round 75/110, elapsed time 3.744\n",
      "[MICE] Starting imputation round 76/110, elapsed time 3.796\n",
      "[MICE] Starting imputation round 77/110, elapsed time 3.844\n",
      "[MICE] Starting imputation round 78/110, elapsed time 3.890\n",
      "[MICE] Starting imputation round 79/110, elapsed time 3.937\n",
      "[MICE] Starting imputation round 80/110, elapsed time 3.986\n",
      "[MICE] Starting imputation round 81/110, elapsed time 4.037\n",
      "[MICE] Starting imputation round 82/110, elapsed time 4.087\n",
      "[MICE] Starting imputation round 83/110, elapsed time 4.137\n",
      "[MICE] Starting imputation round 84/110, elapsed time 4.188\n",
      "[MICE] Starting imputation round 85/110, elapsed time 4.244\n",
      "[MICE] Starting imputation round 86/110, elapsed time 4.308\n",
      "[MICE] Starting imputation round 87/110, elapsed time 4.354\n",
      "[MICE] Starting imputation round 88/110, elapsed time 4.401\n",
      "[MICE] Starting imputation round 89/110, elapsed time 4.448\n",
      "[MICE] Starting imputation round 90/110, elapsed time 4.502\n",
      "[MICE] Starting imputation round 91/110, elapsed time 4.557\n",
      "[MICE] Starting imputation round 92/110, elapsed time 4.611\n",
      "[MICE] Starting imputation round 93/110, elapsed time 4.664\n",
      "[MICE] Starting imputation round 94/110, elapsed time 4.744\n",
      "[MICE] Starting imputation round 95/110, elapsed time 4.800\n",
      "[MICE] Starting imputation round 96/110, elapsed time 4.854\n",
      "[MICE] Starting imputation round 97/110, elapsed time 4.916\n",
      "[MICE] Starting imputation round 98/110, elapsed time 4.965\n",
      "[MICE] Starting imputation round 99/110, elapsed time 5.016\n",
      "[MICE] Starting imputation round 100/110, elapsed time 5.063\n",
      "[MICE] Starting imputation round 101/110, elapsed time 5.112\n",
      "[MICE] Starting imputation round 102/110, elapsed time 5.171\n",
      "[MICE] Starting imputation round 103/110, elapsed time 5.227\n",
      "[MICE] Starting imputation round 104/110, elapsed time 5.279\n",
      "[MICE] Starting imputation round 105/110, elapsed time 5.329\n",
      "[MICE] Starting imputation round 106/110, elapsed time 5.376\n",
      "[MICE] Starting imputation round 107/110, elapsed time 5.428\n",
      "[MICE] Starting imputation round 108/110, elapsed time 5.476\n",
      "[MICE] Starting imputation round 109/110, elapsed time 5.525\n",
      "[MICE] Starting imputation round 110/110, elapsed time 5.578\n"
     ]
    }
   ],
   "source": [
    "from preprocess import preprocessData\n",
    "[Dtrainmat_Diagnosis, Y_FutureDiagnosis, RID, Dtrainmat, Dtrain] = preprocessData()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Embedding, Flatten,BatchNormalization, PReLU, Activation, Lambda, ELU, MaxPooling2D\n",
    "from keras.optimizers import SGD, Adam, RMSprop\n",
    "from keras.constraints import maxnorm\n",
    "import keras.utils\n",
    "\n",
    "Y = Y_FutureDiagnosis.as_matrix()\n",
    "\n",
    "\n",
    "def nn_model():\n",
    "    model = Sequential()\n",
    "    model.add(Dense(500,input_shape=(22,), kernel_initializer = 'he_normal'))\n",
    "    model.add(PReLU())\n",
    "    model.add(Dropout(0.2))\n",
    "    model.add(Dense(300, kernel_initializer=\"he_normal\", kernel_constraint=maxnorm(3)))\n",
    "    model.add(PReLU())\n",
    "    model.add(Dropout(0.2))\n",
    "    #     model.add(Embedding(input_dim=18, output_dim=128))\n",
    "    #     model.add(LSTM(units=128))\n",
    "    model.add(Dense(3, activation='softmax', kernel_initializer='he_normal'))\n",
    "\n",
    "    model.compile(optimizer='Adam',\n",
    "                  loss='categorical_crossentropy',\n",
    "                  metrics=['accuracy'])\n",
    "    return(model)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def createBaggingModels(n):\n",
    "    \"\"\"Helper function to create a dictionary of submodels for use in the bagging training\"\"\"\n",
    "    bagModels = {}\n",
    "    for i in range(n):\n",
    "        name = \"mlp\" + str(i)\n",
    "        print(name)\n",
    "        bagModels[name] = nn_model()\n",
    "        \n",
    "    return bagModels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def run_network(xtrain,ytrain, model=None, epochs=40, batch=64):\n",
    "    start_time = time.time()\n",
    "\n",
    "    one_hot_labels = keras.utils.to_categorical(ytrain - 1, num_classes=3)    \n",
    "    print ('Training model...')\n",
    "    model.fit(xtrain, one_hot_labels, epochs=40, batch_size=64, verbose=2)\n",
    "\n",
    "    print (\"Training duration : {0}\".format(time.time() - start_time))\n",
    "    #score = model.evaluate(X_test, y_test, batch_size=16)\n",
    "\n",
    "    #print (\"Network's test score [loss, accuracy]: {0}\".format(score))\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mlp0\n",
      "mlp1\n",
      "mlp2\n",
      "mlp3\n",
      "mlp4\n",
      "mlp5\n",
      "mlp6\n",
      "mlp7\n",
      "mlp8\n",
      "mlp9\n",
      "[ 1.  1.  1. ...,  1.  1.  1.]\n",
      "Training model: mlp0\n",
      "Training model...\n",
      "Epoch 1/40\n",
      "0s - loss: 0.8162 - acc: 0.6035\n",
      "Epoch 2/40\n",
      "0s - loss: 0.7029 - acc: 0.6564\n",
      "Epoch 3/40\n",
      "0s - loss: 0.6635 - acc: 0.6801\n",
      "Epoch 4/40\n",
      "0s - loss: 0.6318 - acc: 0.7020\n",
      "Epoch 5/40\n",
      "0s - loss: 0.6076 - acc: 0.7171\n",
      "Epoch 6/40\n",
      "0s - loss: 0.5827 - acc: 0.7263\n",
      "Epoch 7/40\n",
      "0s - loss: 0.5690 - acc: 0.7408\n",
      "Epoch 8/40\n",
      "0s - loss: 0.5395 - acc: 0.7527\n",
      "Epoch 9/40\n",
      "0s - loss: 0.5204 - acc: 0.7697\n",
      "Epoch 10/40\n",
      "0s - loss: 0.5032 - acc: 0.7779\n",
      "Epoch 11/40\n",
      "0s - loss: 0.4825 - acc: 0.7941\n",
      "Epoch 12/40\n",
      "0s - loss: 0.4561 - acc: 0.8023\n",
      "Epoch 13/40\n",
      "0s - loss: 0.4446 - acc: 0.8141\n",
      "Epoch 14/40\n",
      "0s - loss: 0.4293 - acc: 0.8159\n",
      "Epoch 15/40\n",
      "0s - loss: 0.4052 - acc: 0.8296\n",
      "Epoch 16/40\n",
      "0s - loss: 0.3868 - acc: 0.8423\n",
      "Epoch 17/40\n",
      "0s - loss: 0.3802 - acc: 0.8390\n",
      "Epoch 18/40\n",
      "0s - loss: 0.3612 - acc: 0.8531\n",
      "Epoch 19/40\n",
      "0s - loss: 0.3490 - acc: 0.8606\n",
      "Epoch 20/40\n",
      "0s - loss: 0.3339 - acc: 0.8623\n",
      "Epoch 21/40\n",
      "0s - loss: 0.3191 - acc: 0.8721\n",
      "Epoch 22/40\n",
      "0s - loss: 0.3203 - acc: 0.8704\n",
      "Epoch 23/40\n",
      "0s - loss: 0.3115 - acc: 0.8764\n",
      "Epoch 24/40\n",
      "0s - loss: 0.2956 - acc: 0.8828\n",
      "Epoch 25/40\n",
      "0s - loss: 0.2785 - acc: 0.8907\n",
      "Epoch 26/40\n",
      "0s - loss: 0.2741 - acc: 0.8911\n",
      "Epoch 27/40\n",
      "0s - loss: 0.2667 - acc: 0.8971\n",
      "Epoch 28/40\n",
      "0s - loss: 0.2582 - acc: 0.9011\n",
      "Epoch 29/40\n",
      "0s - loss: 0.2448 - acc: 0.9071\n",
      "Epoch 30/40\n",
      "0s - loss: 0.2397 - acc: 0.9087\n",
      "Epoch 31/40\n",
      "0s - loss: 0.2434 - acc: 0.9046\n",
      "Epoch 32/40\n",
      "0s - loss: 0.2224 - acc: 0.9166\n",
      "Epoch 33/40\n",
      "0s - loss: 0.2099 - acc: 0.9192\n",
      "Epoch 34/40\n",
      "0s - loss: 0.2112 - acc: 0.9196\n",
      "Epoch 35/40\n",
      "0s - loss: 0.1953 - acc: 0.9261\n",
      "Epoch 36/40\n",
      "0s - loss: 0.2016 - acc: 0.9224\n",
      "Epoch 37/40\n",
      "0s - loss: 0.1978 - acc: 0.9261\n",
      "Epoch 38/40\n",
      "0s - loss: 0.1889 - acc: 0.9293\n",
      "Epoch 39/40\n",
      "0s - loss: 0.1780 - acc: 0.9339\n",
      "Epoch 40/40\n",
      "0s - loss: 0.1728 - acc: 0.9395\n",
      "Training duration : 23.069262981414795\n",
      "Training model: mlp1\n",
      "Training model...\n",
      "Epoch 1/40\n",
      "0s - loss: 0.8139 - acc: 0.6146\n",
      "Epoch 2/40\n",
      "0s - loss: 0.6881 - acc: 0.6753\n",
      "Epoch 3/40\n",
      "0s - loss: 0.6492 - acc: 0.7047\n",
      "Epoch 4/40\n",
      "0s - loss: 0.6163 - acc: 0.7190\n",
      "Epoch 5/40\n",
      "0s - loss: 0.5884 - acc: 0.7333\n",
      "Epoch 6/40\n",
      "0s - loss: 0.5667 - acc: 0.7469\n",
      "Epoch 7/40\n",
      "0s - loss: 0.5478 - acc: 0.7625\n",
      "Epoch 8/40\n",
      "0s - loss: 0.5218 - acc: 0.7684\n",
      "Epoch 9/40\n",
      "0s - loss: 0.5031 - acc: 0.7812\n",
      "Epoch 10/40\n",
      "0s - loss: 0.4786 - acc: 0.7945\n",
      "Epoch 11/40\n",
      "0s - loss: 0.4568 - acc: 0.8075\n",
      "Epoch 12/40\n",
      "0s - loss: 0.4358 - acc: 0.8147\n",
      "Epoch 13/40\n",
      "0s - loss: 0.4346 - acc: 0.8172\n",
      "Epoch 14/40\n",
      "0s - loss: 0.4039 - acc: 0.8300\n",
      "Epoch 15/40\n",
      "0s - loss: 0.3890 - acc: 0.8388\n",
      "Epoch 16/40\n",
      "0s - loss: 0.3828 - acc: 0.8444\n",
      "Epoch 17/40\n",
      "0s - loss: 0.3659 - acc: 0.8515\n",
      "Epoch 18/40\n",
      "0s - loss: 0.3440 - acc: 0.8590\n",
      "Epoch 19/40\n",
      "0s - loss: 0.3407 - acc: 0.8609\n",
      "Epoch 20/40\n",
      "0s - loss: 0.3202 - acc: 0.8723\n",
      "Epoch 21/40\n",
      "0s - loss: 0.3160 - acc: 0.8751\n",
      "Epoch 22/40\n",
      "0s - loss: 0.2910 - acc: 0.8867\n",
      "Epoch 23/40\n",
      "0s - loss: 0.2791 - acc: 0.8974\n",
      "Epoch 24/40\n",
      "0s - loss: 0.2704 - acc: 0.8948\n",
      "Epoch 25/40\n",
      "0s - loss: 0.2693 - acc: 0.8932\n",
      "Epoch 26/40\n",
      "0s - loss: 0.2540 - acc: 0.9040\n",
      "Epoch 27/40\n",
      "0s - loss: 0.2426 - acc: 0.9085\n",
      "Epoch 28/40\n",
      "0s - loss: 0.2322 - acc: 0.9101\n",
      "Epoch 29/40\n",
      "0s - loss: 0.2311 - acc: 0.9138\n",
      "Epoch 30/40\n",
      "0s - loss: 0.2312 - acc: 0.9139\n",
      "Epoch 31/40\n",
      "0s - loss: 0.2144 - acc: 0.9205\n",
      "Epoch 32/40\n",
      "0s - loss: 0.2125 - acc: 0.9224\n",
      "Epoch 33/40\n",
      "0s - loss: 0.2109 - acc: 0.9180\n",
      "Epoch 34/40\n",
      "0s - loss: 0.2041 - acc: 0.9256\n",
      "Epoch 35/40\n",
      "0s - loss: 0.1941 - acc: 0.9277\n",
      "Epoch 36/40\n",
      "0s - loss: 0.1772 - acc: 0.9347\n",
      "Epoch 37/40\n",
      "0s - loss: 0.1776 - acc: 0.9337\n",
      "Epoch 38/40\n",
      "0s - loss: 0.1809 - acc: 0.9316\n",
      "Epoch 39/40\n",
      "0s - loss: 0.1719 - acc: 0.9379\n",
      "Epoch 40/40\n",
      "0s - loss: 0.1679 - acc: 0.9401\n",
      "Training duration : 25.967116117477417\n",
      "Training model: mlp2\n",
      "Training model...\n",
      "Epoch 1/40\n",
      "0s - loss: 0.7976 - acc: 0.6110\n",
      "Epoch 2/40\n",
      "0s - loss: 0.6928 - acc: 0.6674\n",
      "Epoch 3/40\n",
      "0s - loss: 0.6649 - acc: 0.6804\n",
      "Epoch 4/40\n",
      "0s - loss: 0.6324 - acc: 0.7035\n",
      "Epoch 5/40\n",
      "0s - loss: 0.6032 - acc: 0.7181\n",
      "Epoch 6/40\n",
      "0s - loss: 0.5825 - acc: 0.7273\n",
      "Epoch 7/40\n",
      "0s - loss: 0.5582 - acc: 0.7429\n",
      "Epoch 8/40\n",
      "0s - loss: 0.5396 - acc: 0.7605\n",
      "Epoch 9/40\n",
      "0s - loss: 0.5109 - acc: 0.7777\n",
      "Epoch 10/40\n",
      "0s - loss: 0.4933 - acc: 0.7849\n",
      "Epoch 11/40\n",
      "0s - loss: 0.4779 - acc: 0.7918\n",
      "Epoch 12/40\n",
      "0s - loss: 0.4453 - acc: 0.8128\n",
      "Epoch 13/40\n",
      "0s - loss: 0.4330 - acc: 0.8172\n",
      "Epoch 14/40\n",
      "0s - loss: 0.4124 - acc: 0.8289\n",
      "Epoch 15/40\n",
      "0s - loss: 0.4007 - acc: 0.8334\n",
      "Epoch 16/40\n",
      "0s - loss: 0.3822 - acc: 0.8429\n",
      "Epoch 17/40\n",
      "0s - loss: 0.3709 - acc: 0.8539\n",
      "Epoch 18/40\n",
      "0s - loss: 0.3504 - acc: 0.8616\n",
      "Epoch 19/40\n",
      "0s - loss: 0.3370 - acc: 0.8657\n",
      "Epoch 20/40\n",
      "0s - loss: 0.3238 - acc: 0.8670\n",
      "Epoch 21/40\n",
      "0s - loss: 0.3179 - acc: 0.8727\n",
      "Epoch 22/40\n",
      "0s - loss: 0.3017 - acc: 0.8828\n",
      "Epoch 23/40\n",
      "0s - loss: 0.2897 - acc: 0.8853\n",
      "Epoch 24/40\n",
      "0s - loss: 0.2846 - acc: 0.8882\n",
      "Epoch 25/40\n",
      "0s - loss: 0.2670 - acc: 0.8961\n",
      "Epoch 26/40\n",
      "0s - loss: 0.2546 - acc: 0.9019\n",
      "Epoch 27/40\n",
      "0s - loss: 0.2630 - acc: 0.8984\n",
      "Epoch 28/40\n",
      "0s - loss: 0.2574 - acc: 0.9027\n",
      "Epoch 29/40\n",
      "0s - loss: 0.2407 - acc: 0.9112\n",
      "Epoch 30/40\n",
      "0s - loss: 0.2204 - acc: 0.9170\n",
      "Epoch 31/40\n",
      "0s - loss: 0.2284 - acc: 0.9186\n",
      "Epoch 32/40\n",
      "0s - loss: 0.2136 - acc: 0.9190\n",
      "Epoch 33/40\n",
      "0s - loss: 0.2159 - acc: 0.9186\n",
      "Epoch 34/40\n",
      "0s - loss: 0.2000 - acc: 0.9234\n",
      "Epoch 35/40\n",
      "0s - loss: 0.1903 - acc: 0.9297\n",
      "Epoch 36/40\n",
      "0s - loss: 0.1863 - acc: 0.9301\n",
      "Epoch 37/40\n",
      "0s - loss: 0.1808 - acc: 0.9315\n",
      "Epoch 38/40\n",
      "0s - loss: 0.1827 - acc: 0.9338\n",
      "Epoch 39/40\n",
      "0s - loss: 0.1736 - acc: 0.9391\n",
      "Epoch 40/40\n",
      "0s - loss: 0.1690 - acc: 0.9367\n",
      "Training duration : 26.497531175613403\n",
      "Training model: mlp3\n",
      "Training model...\n",
      "Epoch 1/40\n",
      "1s - loss: 0.7930 - acc: 0.6224\n",
      "Epoch 2/40\n",
      "0s - loss: 0.6927 - acc: 0.6621\n",
      "Epoch 3/40\n",
      "0s - loss: 0.6469 - acc: 0.6937\n",
      "Epoch 4/40\n",
      "0s - loss: 0.6287 - acc: 0.7064\n",
      "Epoch 5/40\n",
      "0s - loss: 0.6044 - acc: 0.7222\n",
      "Epoch 6/40\n",
      "0s - loss: 0.5797 - acc: 0.7402\n",
      "Epoch 7/40\n",
      "0s - loss: 0.5526 - acc: 0.7529\n",
      "Epoch 8/40\n",
      "0s - loss: 0.5289 - acc: 0.7675\n",
      "Epoch 9/40\n",
      "0s - loss: 0.5139 - acc: 0.7729\n",
      "Epoch 10/40\n",
      "0s - loss: 0.4879 - acc: 0.7884\n",
      "Epoch 11/40\n",
      "0s - loss: 0.4727 - acc: 0.7963\n",
      "Epoch 12/40\n",
      "0s - loss: 0.4458 - acc: 0.8121\n",
      "Epoch 13/40\n",
      "0s - loss: 0.4291 - acc: 0.8153\n",
      "Epoch 14/40\n",
      "0s - loss: 0.4129 - acc: 0.8274\n",
      "Epoch 15/40\n",
      "0s - loss: 0.3973 - acc: 0.8347\n",
      "Epoch 16/40\n",
      "0s - loss: 0.3818 - acc: 0.8461\n",
      "Epoch 17/40\n",
      "0s - loss: 0.3731 - acc: 0.8482\n",
      "Epoch 18/40\n",
      "0s - loss: 0.3518 - acc: 0.8536\n",
      "Epoch 19/40\n",
      "0s - loss: 0.3345 - acc: 0.8656\n",
      "Epoch 20/40\n",
      "0s - loss: 0.3337 - acc: 0.8653\n",
      "Epoch 21/40\n",
      "0s - loss: 0.3144 - acc: 0.8729\n",
      "Epoch 22/40\n",
      "0s - loss: 0.3049 - acc: 0.8825\n",
      "Epoch 23/40\n",
      "0s - loss: 0.2921 - acc: 0.8846\n",
      "Epoch 24/40\n",
      "0s - loss: 0.2765 - acc: 0.8898\n",
      "Epoch 25/40\n",
      "0s - loss: 0.2756 - acc: 0.8927\n",
      "Epoch 26/40\n",
      "0s - loss: 0.2599 - acc: 0.8989\n",
      "Epoch 27/40\n",
      "0s - loss: 0.2560 - acc: 0.9019\n",
      "Epoch 28/40\n",
      "0s - loss: 0.2474 - acc: 0.9047\n",
      "Epoch 29/40\n",
      "0s - loss: 0.2324 - acc: 0.9114\n",
      "Epoch 30/40\n",
      "0s - loss: 0.2213 - acc: 0.9180\n",
      "Epoch 31/40\n",
      "0s - loss: 0.2126 - acc: 0.9179\n",
      "Epoch 32/40\n",
      "0s - loss: 0.2134 - acc: 0.9190\n",
      "Epoch 33/40\n",
      "0s - loss: 0.2076 - acc: 0.9233\n",
      "Epoch 34/40\n",
      "0s - loss: 0.1964 - acc: 0.9268\n",
      "Epoch 35/40\n",
      "0s - loss: 0.1968 - acc: 0.9263\n",
      "Epoch 36/40\n",
      "0s - loss: 0.1858 - acc: 0.9300\n",
      "Epoch 37/40\n",
      "0s - loss: 0.1880 - acc: 0.9319\n",
      "Epoch 38/40\n",
      "0s - loss: 0.1723 - acc: 0.9386\n",
      "Epoch 39/40\n",
      "0s - loss: 0.1753 - acc: 0.9370\n",
      "Epoch 40/40\n",
      "0s - loss: 0.1674 - acc: 0.9407\n",
      "Training duration : 26.84109902381897\n",
      "Training model: mlp4\n",
      "Training model...\n",
      "Epoch 1/40\n",
      "1s - loss: 0.8049 - acc: 0.6171\n",
      "Epoch 2/40\n",
      "0s - loss: 0.7033 - acc: 0.6684\n",
      "Epoch 3/40\n",
      "0s - loss: 0.6638 - acc: 0.6943\n",
      "Epoch 4/40\n",
      "0s - loss: 0.6339 - acc: 0.7112\n",
      "Epoch 5/40\n",
      "0s - loss: 0.6039 - acc: 0.7253\n",
      "Epoch 6/40\n",
      "0s - loss: 0.5846 - acc: 0.7324\n",
      "Epoch 7/40\n",
      "0s - loss: 0.5608 - acc: 0.7501\n",
      "Epoch 8/40\n",
      "0s - loss: 0.5404 - acc: 0.7621\n",
      "Epoch 9/40\n",
      "0s - loss: 0.5231 - acc: 0.7725\n",
      "Epoch 10/40\n",
      "0s - loss: 0.4982 - acc: 0.7865\n",
      "Epoch 11/40\n",
      "0s - loss: 0.4749 - acc: 0.7986\n",
      "Epoch 12/40\n",
      "0s - loss: 0.4542 - acc: 0.8097\n",
      "Epoch 13/40\n",
      "0s - loss: 0.4478 - acc: 0.8051\n",
      "Epoch 14/40\n",
      "0s - loss: 0.4092 - acc: 0.8359\n",
      "Epoch 15/40\n",
      "0s - loss: 0.4022 - acc: 0.8343\n",
      "Epoch 16/40\n",
      "0s - loss: 0.3801 - acc: 0.8435\n",
      "Epoch 17/40\n",
      "0s - loss: 0.3826 - acc: 0.8395\n",
      "Epoch 18/40\n",
      "0s - loss: 0.3522 - acc: 0.8526\n",
      "Epoch 19/40\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0s - loss: 0.3477 - acc: 0.8582\n",
      "Epoch 20/40\n",
      "0s - loss: 0.3317 - acc: 0.8650\n",
      "Epoch 21/40\n",
      "0s - loss: 0.3156 - acc: 0.8749\n",
      "Epoch 22/40\n",
      "0s - loss: 0.3014 - acc: 0.8772\n",
      "Epoch 23/40\n",
      "0s - loss: 0.2890 - acc: 0.8841\n",
      "Epoch 24/40\n",
      "0s - loss: 0.2852 - acc: 0.8866\n",
      "Epoch 25/40\n",
      "0s - loss: 0.2753 - acc: 0.8932\n",
      "Epoch 26/40\n",
      "0s - loss: 0.2692 - acc: 0.8957\n",
      "Epoch 27/40\n",
      "0s - loss: 0.2542 - acc: 0.9019\n",
      "Epoch 28/40\n",
      "0s - loss: 0.2479 - acc: 0.9047\n",
      "Epoch 29/40\n",
      "0s - loss: 0.2380 - acc: 0.9119\n",
      "Epoch 30/40\n",
      "0s - loss: 0.2272 - acc: 0.9132\n",
      "Epoch 31/40\n",
      "0s - loss: 0.2307 - acc: 0.9098\n",
      "Epoch 32/40\n",
      "0s - loss: 0.2141 - acc: 0.9190\n",
      "Epoch 33/40\n",
      "0s - loss: 0.2110 - acc: 0.9218\n",
      "Epoch 34/40\n",
      "0s - loss: 0.2007 - acc: 0.9256\n",
      "Epoch 35/40\n",
      "0s - loss: 0.1925 - acc: 0.9322\n",
      "Epoch 36/40\n",
      "0s - loss: 0.1947 - acc: 0.9309\n",
      "Epoch 37/40\n",
      "0s - loss: 0.1737 - acc: 0.9347\n",
      "Epoch 38/40\n",
      "0s - loss: 0.1812 - acc: 0.9319\n",
      "Epoch 39/40\n",
      "0s - loss: 0.1694 - acc: 0.9360\n",
      "Epoch 40/40\n",
      "0s - loss: 0.1596 - acc: 0.9461\n",
      "Training duration : 26.011595010757446\n",
      "Training model: mlp5\n",
      "Training model...\n",
      "Epoch 1/40\n",
      "1s - loss: 0.7981 - acc: 0.6107\n",
      "Epoch 2/40\n",
      "0s - loss: 0.6920 - acc: 0.6716\n",
      "Epoch 3/40\n",
      "0s - loss: 0.6542 - acc: 0.6852\n",
      "Epoch 4/40\n",
      "0s - loss: 0.6196 - acc: 0.7101\n",
      "Epoch 5/40\n",
      "0s - loss: 0.6046 - acc: 0.7209\n",
      "Epoch 6/40\n",
      "0s - loss: 0.5674 - acc: 0.7441\n",
      "Epoch 7/40\n",
      "0s - loss: 0.5468 - acc: 0.7526\n",
      "Epoch 8/40\n",
      "0s - loss: 0.5283 - acc: 0.7634\n",
      "Epoch 9/40\n",
      "0s - loss: 0.5025 - acc: 0.7798\n",
      "Epoch 10/40\n",
      "0s - loss: 0.4896 - acc: 0.7839\n",
      "Epoch 11/40\n",
      "0s - loss: 0.4674 - acc: 0.7954\n",
      "Epoch 12/40\n",
      "0s - loss: 0.4462 - acc: 0.8110\n",
      "Epoch 13/40\n",
      "0s - loss: 0.4322 - acc: 0.8144\n",
      "Epoch 14/40\n",
      "0s - loss: 0.4101 - acc: 0.8281\n",
      "Epoch 15/40\n",
      "0s - loss: 0.3949 - acc: 0.8360\n",
      "Epoch 16/40\n",
      "0s - loss: 0.3807 - acc: 0.8439\n",
      "Epoch 17/40\n",
      "0s - loss: 0.3616 - acc: 0.8553\n",
      "Epoch 18/40\n",
      "0s - loss: 0.3482 - acc: 0.8615\n",
      "Epoch 19/40\n",
      "0s - loss: 0.3332 - acc: 0.8644\n",
      "Epoch 20/40\n",
      "0s - loss: 0.3260 - acc: 0.8696\n",
      "Epoch 21/40\n",
      "0s - loss: 0.3109 - acc: 0.8767\n",
      "Epoch 22/40\n",
      "0s - loss: 0.2994 - acc: 0.8853\n",
      "Epoch 23/40\n",
      "0s - loss: 0.2921 - acc: 0.8894\n",
      "Epoch 24/40\n",
      "0s - loss: 0.2788 - acc: 0.8910\n",
      "Epoch 25/40\n",
      "0s - loss: 0.2705 - acc: 0.8970\n",
      "Epoch 26/40\n",
      "0s - loss: 0.2590 - acc: 0.9015\n",
      "Epoch 27/40\n",
      "0s - loss: 0.2456 - acc: 0.9074\n",
      "Epoch 28/40\n",
      "0s - loss: 0.2475 - acc: 0.9072\n",
      "Epoch 29/40\n",
      "0s - loss: 0.2334 - acc: 0.9125\n",
      "Epoch 30/40\n",
      "0s - loss: 0.2215 - acc: 0.9187\n",
      "Epoch 31/40\n",
      "0s - loss: 0.2188 - acc: 0.9154\n",
      "Epoch 32/40\n",
      "0s - loss: 0.2094 - acc: 0.9220\n",
      "Epoch 33/40\n",
      "0s - loss: 0.2044 - acc: 0.9244\n",
      "Epoch 34/40\n",
      "0s - loss: 0.1984 - acc: 0.9265\n",
      "Epoch 35/40\n",
      "0s - loss: 0.1986 - acc: 0.9250\n",
      "Epoch 36/40\n",
      "0s - loss: 0.1889 - acc: 0.9313\n",
      "Epoch 37/40\n",
      "0s - loss: 0.1883 - acc: 0.9288\n",
      "Epoch 38/40\n",
      "0s - loss: 0.1953 - acc: 0.9275\n",
      "Epoch 39/40\n",
      "0s - loss: 0.1752 - acc: 0.9338\n",
      "Epoch 40/40\n",
      "0s - loss: 0.1763 - acc: 0.9326\n",
      "Training duration : 26.972293853759766\n",
      "Training model: mlp6\n",
      "Training model...\n",
      "Epoch 1/40\n",
      "0s - loss: 0.7992 - acc: 0.6106\n",
      "Epoch 2/40\n",
      "0s - loss: 0.6962 - acc: 0.6686\n",
      "Epoch 3/40\n",
      "0s - loss: 0.6627 - acc: 0.6841\n",
      "Epoch 4/40\n",
      "0s - loss: 0.6359 - acc: 0.7057\n",
      "Epoch 5/40\n",
      "0s - loss: 0.6201 - acc: 0.7193\n",
      "Epoch 6/40\n",
      "0s - loss: 0.5838 - acc: 0.7330\n",
      "Epoch 7/40\n",
      "0s - loss: 0.5558 - acc: 0.7473\n",
      "Epoch 8/40\n",
      "0s - loss: 0.5366 - acc: 0.7637\n",
      "Epoch 9/40\n",
      "0s - loss: 0.5197 - acc: 0.7748\n",
      "Epoch 10/40\n",
      "0s - loss: 0.4914 - acc: 0.7831\n",
      "Epoch 11/40\n",
      "0s - loss: 0.4816 - acc: 0.7906\n",
      "Epoch 12/40\n",
      "0s - loss: 0.4595 - acc: 0.8049\n",
      "Epoch 13/40\n",
      "0s - loss: 0.4403 - acc: 0.8125\n",
      "Epoch 14/40\n",
      "0s - loss: 0.4152 - acc: 0.8279\n",
      "Epoch 15/40\n",
      "0s - loss: 0.4082 - acc: 0.8303\n",
      "Epoch 16/40\n",
      "0s - loss: 0.3874 - acc: 0.8425\n",
      "Epoch 17/40\n",
      "0s - loss: 0.3689 - acc: 0.8505\n",
      "Epoch 18/40\n",
      "0s - loss: 0.3574 - acc: 0.8534\n",
      "Epoch 19/40\n",
      "0s - loss: 0.3469 - acc: 0.8600\n",
      "Epoch 20/40\n",
      "0s - loss: 0.3343 - acc: 0.8669\n",
      "Epoch 21/40\n",
      "0s - loss: 0.3213 - acc: 0.8745\n",
      "Epoch 22/40\n",
      "0s - loss: 0.3148 - acc: 0.8813\n",
      "Epoch 23/40\n",
      "0s - loss: 0.2985 - acc: 0.8829\n",
      "Epoch 24/40\n",
      "0s - loss: 0.2889 - acc: 0.8889\n",
      "Epoch 25/40\n",
      "0s - loss: 0.2771 - acc: 0.8905\n",
      "Epoch 26/40\n",
      "0s - loss: 0.2691 - acc: 0.8936\n",
      "Epoch 27/40\n",
      "0s - loss: 0.2560 - acc: 0.9074\n",
      "Epoch 28/40\n",
      "0s - loss: 0.2462 - acc: 0.9104\n",
      "Epoch 29/40\n",
      "0s - loss: 0.2358 - acc: 0.9094\n",
      "Epoch 30/40\n",
      "0s - loss: 0.2380 - acc: 0.9109\n",
      "Epoch 31/40\n",
      "0s - loss: 0.2307 - acc: 0.9101\n",
      "Epoch 32/40\n",
      "0s - loss: 0.2143 - acc: 0.9214\n",
      "Epoch 33/40\n",
      "0s - loss: 0.2014 - acc: 0.9228\n",
      "Epoch 34/40\n",
      "0s - loss: 0.2122 - acc: 0.9183\n",
      "Epoch 35/40\n",
      "0s - loss: 0.2039 - acc: 0.9249\n",
      "Epoch 36/40\n",
      "0s - loss: 0.1896 - acc: 0.9288\n",
      "Epoch 37/40\n",
      "0s - loss: 0.1878 - acc: 0.9309\n",
      "Epoch 38/40\n",
      "0s - loss: 0.1849 - acc: 0.9307\n",
      "Epoch 39/40\n",
      "0s - loss: 0.1762 - acc: 0.9345\n",
      "Epoch 40/40\n",
      "0s - loss: 0.1680 - acc: 0.9386\n",
      "Training duration : 25.707562923431396\n",
      "Training model: mlp7\n",
      "Training model...\n",
      "Epoch 1/40\n",
      "0s - loss: 0.8070 - acc: 0.6117\n",
      "Epoch 2/40\n",
      "0s - loss: 0.6937 - acc: 0.6598\n",
      "Epoch 3/40\n",
      "0s - loss: 0.6620 - acc: 0.6839\n",
      "Epoch 4/40\n",
      "0s - loss: 0.6326 - acc: 0.6995\n",
      "Epoch 5/40\n",
      "0s - loss: 0.6031 - acc: 0.7209\n",
      "Epoch 6/40\n",
      "0s - loss: 0.5882 - acc: 0.7280\n",
      "Epoch 7/40\n",
      "0s - loss: 0.5600 - acc: 0.7457\n",
      "Epoch 8/40\n",
      "0s - loss: 0.5294 - acc: 0.7694\n",
      "Epoch 9/40\n",
      "0s - loss: 0.5169 - acc: 0.7690\n",
      "Epoch 10/40\n",
      "0s - loss: 0.4887 - acc: 0.7909\n",
      "Epoch 11/40\n",
      "0s - loss: 0.4776 - acc: 0.7929\n",
      "Epoch 12/40\n",
      "0s - loss: 0.4533 - acc: 0.8048\n",
      "Epoch 13/40\n",
      "0s - loss: 0.4366 - acc: 0.8229\n",
      "Epoch 14/40\n",
      "0s - loss: 0.4103 - acc: 0.8260\n",
      "Epoch 15/40\n",
      "0s - loss: 0.3950 - acc: 0.8366\n",
      "Epoch 16/40\n",
      "0s - loss: 0.3870 - acc: 0.8416\n",
      "Epoch 17/40\n",
      "0s - loss: 0.3592 - acc: 0.8540\n",
      "Epoch 18/40\n",
      "0s - loss: 0.3476 - acc: 0.8569\n",
      "Epoch 19/40\n",
      "0s - loss: 0.3458 - acc: 0.8639\n",
      "Epoch 20/40\n",
      "0s - loss: 0.3258 - acc: 0.8686\n",
      "Epoch 21/40\n",
      "0s - loss: 0.3112 - acc: 0.8727\n",
      "Epoch 22/40\n",
      "0s - loss: 0.3026 - acc: 0.8803\n",
      "Epoch 23/40\n",
      "0s - loss: 0.2816 - acc: 0.8913\n",
      "Epoch 24/40\n",
      "0s - loss: 0.2765 - acc: 0.8923\n",
      "Epoch 25/40\n",
      "0s - loss: 0.2701 - acc: 0.8960\n",
      "Epoch 26/40\n",
      "0s - loss: 0.2556 - acc: 0.8979\n",
      "Epoch 27/40\n",
      "0s - loss: 0.2482 - acc: 0.9065\n",
      "Epoch 28/40\n",
      "0s - loss: 0.2407 - acc: 0.9076\n",
      "Epoch 29/40\n",
      "0s - loss: 0.2322 - acc: 0.9084\n",
      "Epoch 30/40\n",
      "0s - loss: 0.2341 - acc: 0.9136\n",
      "Epoch 31/40\n",
      "0s - loss: 0.2122 - acc: 0.9198\n",
      "Epoch 32/40\n",
      "0s - loss: 0.2132 - acc: 0.9182\n",
      "Epoch 33/40\n",
      "0s - loss: 0.2036 - acc: 0.9256\n",
      "Epoch 34/40\n",
      "0s - loss: 0.1963 - acc: 0.9255\n",
      "Epoch 35/40\n",
      "0s - loss: 0.1853 - acc: 0.9291\n",
      "Epoch 36/40\n",
      "0s - loss: 0.1963 - acc: 0.9277\n",
      "Epoch 37/40\n",
      "0s - loss: 0.1745 - acc: 0.9373\n",
      "Epoch 38/40\n",
      "0s - loss: 0.1815 - acc: 0.9316\n",
      "Epoch 39/40\n",
      "0s - loss: 0.1639 - acc: 0.9445\n",
      "Epoch 40/40\n",
      "0s - loss: 0.1701 - acc: 0.9364\n",
      "Training duration : 24.942129135131836\n",
      "Training model: mlp8\n",
      "Training model...\n",
      "Epoch 1/40\n",
      "1s - loss: 0.8167 - acc: 0.6091\n",
      "Epoch 2/40\n",
      "0s - loss: 0.7020 - acc: 0.6623\n",
      "Epoch 3/40\n",
      "0s - loss: 0.6726 - acc: 0.6838\n",
      "Epoch 4/40\n",
      "0s - loss: 0.6357 - acc: 0.7073\n",
      "Epoch 5/40\n",
      "0s - loss: 0.6021 - acc: 0.7286\n",
      "Epoch 6/40\n",
      "0s - loss: 0.5826 - acc: 0.7348\n",
      "Epoch 7/40\n",
      "0s - loss: 0.5533 - acc: 0.7511\n",
      "Epoch 8/40\n",
      "0s - loss: 0.5409 - acc: 0.7634\n",
      "Epoch 9/40\n",
      "0s - loss: 0.5146 - acc: 0.7786\n",
      "Epoch 10/40\n",
      "0s - loss: 0.4900 - acc: 0.7899\n",
      "Epoch 11/40\n",
      "0s - loss: 0.4774 - acc: 0.7980\n",
      "Epoch 12/40\n",
      "0s - loss: 0.4505 - acc: 0.8116\n",
      "Epoch 13/40\n",
      "0s - loss: 0.4335 - acc: 0.8147\n",
      "Epoch 14/40\n",
      "0s - loss: 0.4159 - acc: 0.8305\n",
      "Epoch 15/40\n",
      "0s - loss: 0.4010 - acc: 0.8400\n",
      "Epoch 16/40\n",
      "0s - loss: 0.3813 - acc: 0.8496\n",
      "Epoch 17/40\n",
      "0s - loss: 0.3671 - acc: 0.8549\n",
      "Epoch 18/40\n",
      "0s - loss: 0.3589 - acc: 0.8526\n",
      "Epoch 19/40\n",
      "0s - loss: 0.3425 - acc: 0.8604\n",
      "Epoch 20/40\n",
      "0s - loss: 0.3217 - acc: 0.8705\n",
      "Epoch 21/40\n",
      "0s - loss: 0.3153 - acc: 0.8775\n",
      "Epoch 22/40\n",
      "0s - loss: 0.3069 - acc: 0.8755\n",
      "Epoch 23/40\n",
      "0s - loss: 0.2912 - acc: 0.8892\n",
      "Epoch 24/40\n",
      "0s - loss: 0.2823 - acc: 0.8879\n",
      "Epoch 25/40\n",
      "0s - loss: 0.2760 - acc: 0.8933\n",
      "Epoch 26/40\n",
      "0s - loss: 0.2708 - acc: 0.8949\n",
      "Epoch 27/40\n",
      "0s - loss: 0.2524 - acc: 0.9066\n",
      "Epoch 28/40\n",
      "0s - loss: 0.2532 - acc: 0.9031\n",
      "Epoch 29/40\n",
      "0s - loss: 0.2414 - acc: 0.9097\n",
      "Epoch 30/40\n",
      "0s - loss: 0.2230 - acc: 0.9161\n",
      "Epoch 31/40\n",
      "0s - loss: 0.2194 - acc: 0.9199\n",
      "Epoch 32/40\n",
      "0s - loss: 0.2229 - acc: 0.9145\n",
      "Epoch 33/40\n",
      "0s - loss: 0.2075 - acc: 0.9204\n",
      "Epoch 34/40\n",
      "0s - loss: 0.2062 - acc: 0.9233\n",
      "Epoch 35/40\n",
      "0s - loss: 0.1914 - acc: 0.9301\n",
      "Epoch 36/40\n",
      "0s - loss: 0.1887 - acc: 0.9319\n",
      "Epoch 37/40\n",
      "0s - loss: 0.1949 - acc: 0.9243\n",
      "Epoch 38/40\n",
      "0s - loss: 0.1804 - acc: 0.9351\n",
      "Epoch 39/40\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0s - loss: 0.1687 - acc: 0.9411\n",
      "Epoch 40/40\n",
      "0s - loss: 0.1611 - acc: 0.9423\n",
      "Training duration : 26.283725023269653\n",
      "Training model: mlp9\n",
      "Training model...\n",
      "Epoch 1/40\n",
      "1s - loss: 0.8091 - acc: 0.6082\n",
      "Epoch 2/40\n",
      "0s - loss: 0.6924 - acc: 0.6658\n",
      "Epoch 3/40\n",
      "0s - loss: 0.6541 - acc: 0.6841\n",
      "Epoch 4/40\n",
      "0s - loss: 0.6245 - acc: 0.7102\n",
      "Epoch 5/40\n",
      "0s - loss: 0.6001 - acc: 0.7267\n",
      "Epoch 6/40\n",
      "0s - loss: 0.5844 - acc: 0.7387\n",
      "Epoch 7/40\n",
      "0s - loss: 0.5504 - acc: 0.7517\n",
      "Epoch 8/40\n",
      "0s - loss: 0.5309 - acc: 0.7653\n",
      "Epoch 9/40\n",
      "0s - loss: 0.5114 - acc: 0.7750\n",
      "Epoch 10/40\n",
      "0s - loss: 0.4901 - acc: 0.7954\n",
      "Epoch 11/40\n",
      "0s - loss: 0.4733 - acc: 0.7976\n",
      "Epoch 12/40\n",
      "0s - loss: 0.4540 - acc: 0.8121\n",
      "Epoch 13/40\n",
      "0s - loss: 0.4366 - acc: 0.8184\n",
      "Epoch 14/40\n",
      "0s - loss: 0.4247 - acc: 0.8185\n",
      "Epoch 15/40\n",
      "0s - loss: 0.4010 - acc: 0.8372\n",
      "Epoch 16/40\n",
      "0s - loss: 0.3925 - acc: 0.8346\n",
      "Epoch 17/40\n",
      "0s - loss: 0.3799 - acc: 0.8407\n",
      "Epoch 18/40\n",
      "0s - loss: 0.3534 - acc: 0.8561\n",
      "Epoch 19/40\n",
      "0s - loss: 0.3404 - acc: 0.8622\n",
      "Epoch 20/40\n",
      "0s - loss: 0.3286 - acc: 0.8648\n",
      "Epoch 21/40\n",
      "0s - loss: 0.3180 - acc: 0.8764\n",
      "Epoch 22/40\n",
      "0s - loss: 0.3097 - acc: 0.8805\n",
      "Epoch 23/40\n",
      "0s - loss: 0.3048 - acc: 0.8848\n",
      "Epoch 24/40\n",
      "0s - loss: 0.2867 - acc: 0.8900\n",
      "Epoch 25/40\n",
      "0s - loss: 0.2702 - acc: 0.8946\n",
      "Epoch 26/40\n",
      "0s - loss: 0.2586 - acc: 0.9037\n",
      "Epoch 27/40\n",
      "0s - loss: 0.2580 - acc: 0.8949\n",
      "Epoch 28/40\n",
      "0s - loss: 0.2452 - acc: 0.9028\n",
      "Epoch 29/40\n",
      "0s - loss: 0.2367 - acc: 0.9120\n",
      "Epoch 30/40\n",
      "0s - loss: 0.2381 - acc: 0.9100\n",
      "Epoch 31/40\n",
      "0s - loss: 0.2240 - acc: 0.9147\n",
      "Epoch 32/40\n",
      "0s - loss: 0.2195 - acc: 0.9154\n",
      "Epoch 33/40\n",
      "0s - loss: 0.2070 - acc: 0.9223\n",
      "Epoch 34/40\n",
      "0s - loss: 0.2002 - acc: 0.9224\n",
      "Epoch 35/40\n",
      "0s - loss: 0.2110 - acc: 0.9179\n",
      "Epoch 36/40\n",
      "0s - loss: 0.1896 - acc: 0.9313\n",
      "Epoch 37/40\n",
      "0s - loss: 0.1838 - acc: 0.9331\n",
      "Epoch 38/40\n",
      "0s - loss: 0.1782 - acc: 0.9363\n",
      "Epoch 39/40\n",
      "0s - loss: 0.1846 - acc: 0.9290\n",
      "Epoch 40/40\n",
      "0s - loss: 0.1741 - acc: 0.9341\n",
      "Training duration : 26.70605182647705\n"
     ]
    }
   ],
   "source": [
    "# Train all the models\n",
    "subModelCount = 10\n",
    "models = createBaggingModels(subModelCount)\n",
    "print(Y)\n",
    "import time\n",
    "X_train = Dtrainmat_Diagnosis\n",
    "toTrain = models.keys()\n",
    "bootstrapTrainSize = len(X_train) * 1\n",
    "for k in toTrain:\n",
    "    print(\"Training model: \" + k)\n",
    "    trainNdx = np.random.choice(range(len(X_train)), int(bootstrapTrainSize))\n",
    "    m = run_network(X_train[trainNdx],Y[trainNdx], models[k])\n",
    "    models[k].save_weights(\"model-\" + k + \".hdf5\", overwrite=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running predictions on test set samples...\n",
      "Model: mlp0\n",
      "(110, 3)\n",
      "Model: mlp1\n",
      "(110, 3)\n",
      "Model: mlp2\n",
      "(110, 3)\n",
      "Model: mlp3\n",
      "(110, 3)\n",
      "Model: mlp4\n",
      "(110, 3)\n",
      "Model: mlp5\n",
      "(110, 3)\n",
      "Model: mlp6\n",
      "(110, 3)\n",
      "Model: mlp7\n",
      "(110, 3)\n",
      "Model: mlp8\n",
      "(110, 3)\n",
      "Model: mlp9\n",
      "(110, 3)\n",
      "Evaluate predictions\n",
      "Diagnosis:\n",
      "mAUC = 0.645\n",
      "BAC = 0.566\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "\n",
    "    \n",
    "str_exp = '/Users/danielwu/Dropbox/Documents/CSCI5525/Project/'\n",
    "S = pd.read_csv(os.path.join(str_exp, 'IntermediateData',\n",
    "                             'ToPredict.csv'), header=None)\n",
    "S = S.values\n",
    "\n",
    "Dtestmat = np.zeros((len(S), Dtrainmat.shape[1]))\n",
    "# y_Dtestmat = np.zeros((len(S),1))\n",
    "for i in range(len(S)):\n",
    "    idx_S = RID.values == S[i]\n",
    "    Dtestmat[i, :] = Dtrainmat[np.where(idx_S)[0][-1], :]\n",
    "#     y_Dtestmat[i] = Dtrainmat[np.where(idx_S)[0][-1], :]\n",
    "\n",
    "# mt = []\n",
    "# st = []\n",
    "# for i in range(Dtrainmat.shape[1]):\n",
    "#     mt.append(np.nanmean(Dtestmat[:, i]))\n",
    "#     st.append(np.nanstd(Dtestmat[:, i]))\n",
    "#     Dtestmat[np.isnan(Dtestmat[:, i]), i] = mt[i]\n",
    "#     Dtestmat[:, i] = (Dtestmat[:, i] - mt[i]) / st[i]\n",
    "\n",
    "    \n",
    "print(\"Running predictions on test set samples...\")\n",
    "results = np.zeros((len(S),3))\n",
    "#print(np.shape(results))\n",
    "for k in models.keys():\n",
    "    print(\"Model: \" + k)\n",
    "    p = models[k].predict(Dtestmat)\n",
    "    results += p\n",
    "    print(np.shape(p))\n",
    "\n",
    "results = results / subModelCount\n",
    "# print(results)\n",
    "\n",
    "# Write ouput format to files\n",
    "o = np.column_stack((S, S, S, results))\n",
    "count = 0\n",
    "years = [str(a) for a in range(2010, 2018)]\n",
    "months = [str(a).zfill(2) for a in range(1, 13)]\n",
    "ym = [y + '-' + mo for y in years for mo in months]\n",
    "ym = ym[4:-8]\n",
    "nr_pred = len(ym)\n",
    "o1 = np.zeros((o.shape[0] * nr_pred, o.shape[1]))\n",
    "ym1 = [a for b in range(0, len(S)) for a in ym]\n",
    "for i in range(len(o)):\n",
    "    o1[count:count + nr_pred] = o[i]\n",
    "    o1[count:count + nr_pred, 1] = range(1, nr_pred + 1)\n",
    "    count = count + nr_pred\n",
    "\n",
    "output = pd.DataFrame(o1, columns=['RID', 'Forecast Month', 'Forecast Date','CN relative probability', 'MCI relative probability', 'AD relative probability'])\n",
    "output['Forecast Month'] = output['Forecast Month'].astype(int)\n",
    "output['Forecast Date'] = ym1\n",
    "\n",
    "str_out_final = os.path.join(str_exp, 'IntermediateData', 'TADPOLE_Submission_Leaderboard_NeuralNet.csv')\n",
    "output.to_csv(str_out_final, header=True, index=False)\n",
    "\n",
    "print('Evaluate predictions')\n",
    "R = pd.read_csv('./TADPOLE_LB4.csv')\n",
    "import evalOneSubmission as eos\n",
    "mAUC, bca = eos.evalOneSub(R, output)\n",
    "\n",
    "print('Diagnosis:')\n",
    "print('mAUC = ' + \"%0.3f\" % mAUC)\n",
    "print('BAC = ' + \"%0.3f\" % bca)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def BaggingAverage(results, verbose = False):\n",
    "    \"\"\"Average the probabilities for a given test sample across models to finalize the prediction\"\"\" \n",
    "    correct = 0.0\n",
    "    missedTests = []\n",
    "    predictionMisses = []\n",
    "    for i in range(samples):\n",
    "        gs = np.argmax(y_test[i])\n",
    "        ra = np.zeros(3)\n",
    "        for r in reuslts:\n",
    "            ra = += r[i]\n",
    "        f = ra/ len(results)\n",
    "        prd = np.argmax(f)\n",
    "        frnd = np.round(f,5)\n",
    "    if(prd == gs):\n",
    "        correct += 1\n",
    "    else:\n",
    "        if(verbose):\n",
    "            for r in results:\n",
    "                print(np.round(r[i], 5))\n",
    "            print(\"Agg: \" + str(frnd))\n",
    "            print(\"Predicted: \" + str(prd))\n",
    "            print(\"GoldStd:   \" + str(gs))\n",
    "        missedTests.append(X_test[i])\n",
    "        predictionMisses.append(prd)\n",
    "    \n",
    "    return correct\n",
    "\n",
    "def BaggingVote(results, verbose = False):\n",
    "    \"\"\"Vote based on the probabilities for a given test sample across models to finalize the prediction\"\"\"\n",
    "    correct = 0.0\n",
    "    \n",
    "    missedTests = []\n",
    "    predictionMisses = []\n",
    "    for i in range(samples):\n",
    "        #print(y_test[i])\n",
    "        gs = np.argmax(y_test[i])\n",
    "        votes = np.zeros(3)\n",
    "        for r in results:\n",
    "            # Use argmax to get the primary vote from this model and increment the cooresponding slot \n",
    "            # in the aggregated result\n",
    "            vote = np.argmax(r[i])\n",
    "            #print(vote)\n",
    "            votes[vote] += 1\n",
    "        # Use argmax again to extract the slot that has the most votes.\n",
    "        prd = np.argmax(votes)\n",
    "\n",
    "        if(prd == gs):\n",
    "            correct += 1\n",
    "        else:\n",
    "            if(verbose):\n",
    "                for r in results:\n",
    "                    print(np.round(r[i], 5))\n",
    "                print(\"Votes: \" + str(votes))\n",
    "                print(\"Predicted: \" + str(prd))\n",
    "                print(\"GoldStd:   \" + str(gs))\n",
    "            missedTests.append(X_test[i])\n",
    "            predictionMisses.append(prd)\n",
    "    \n",
    "    return correct"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Write ouput format to files\n",
    "o=np.column_stack((S,S,S,p))\n",
    "count=0\n",
    "years=[str(a) for a in range(2010,2018)]\n",
    "months=[str(a).zfill(2) for a in range(1,13)]\n",
    "ym=[y + '-' + mo for y in years for mo in months ]\n",
    "ym=ym[4:-8]\n",
    "nr_pred=len(ym)\n",
    "o1 = np.zeros((o.shape[0]*nr_pred,o.shape[1]))\n",
    "ym1 = [a for b in range(0, len(S)) for a in ym ]\n",
    "for i in range(len(o)):\n",
    "    o1[count:count+nr_pred]=o[i]\n",
    "    o1[count:count+nr_pred,1]=range(1,nr_pred+1)\n",
    "    count=count+nr_pred\n",
    "    \n",
    "\n",
    "output=pd.DataFrame(o1, columns=['RID','Forecast Month','Forecast Date','CN relative probability','MCI relative probability','AD relative probability'])\n",
    "output['Forecast Month'] = output['Forecast Month'].astype(int)\n",
    "output['Forecast Date'] = ym1\n",
    "\n",
    "str_out_final = os.path.join(str_exp, 'IntermediateData','TADPOLE_Submission_Leaderboard_BenchmarkSVM.csv')\n",
    "output.to_csv(str_out_final,header=True,index=False)\n",
    "\n",
    "\n",
    "print('Evaluate predictions')\n",
    "R=pd.read_csv('./TADPOLE_LB4.csv')\n",
    "import evalOneSubmission as eos\n",
    "mAUC, bca = eos.evalOneSub(R,output)\n",
    "\n",
    "print('Diagnosis:')\n",
    "print('mAUC = ' + \"%0.3f\" % mAUC,)\n",
    "print ('BAC = ' + \"%0.3f\" % bca)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# What is our percentage success from averaging?\n",
    "print (\"Bagging w/ Averaging\")\n",
    "print (\"--------------------\")\n",
    "correct = BaggingAverage(results)\n",
    "print (\"Misses: \" + str(samples - correct))\n",
    "print correct/samples\n",
    "\n",
    "# Separator...\n",
    "print\n",
    "print\n",
    "\n",
    "# What is our percentage success from voting?\n",
    "print (\"Bagging w/ Voting\")\n",
    "print (\"-----------------\")\n",
    "correctVote = BaggingVote(results)\n",
    "print (\"Misses: \" + str(samples - correctVote))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}