nn_forward_train.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "import time\n",
    "import numpy as np\n",
    "import theano\n",
    "import theano.tensor as T\n",
    "import lasagne\n",
    "\n",
    "import data_io_func\n",
    "\n",
    "theano.config.floatX='float32'\n",
    "# set a random numberinitialization seed to make code reproducible:\n",
    "lasagne.random.set_rng(np.random.RandomState(seed=1)) # for lasagne\n",
    "np.random.seed(seed=1) # for shuffling training examples"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "First we need to load some data. Here we'll use MHC class 1 binding data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2471, 9, 21)\n",
      "(618, 9, 21)\n"
     ]
    }
   ],
   "source": [
    "MAX_PEP_SEQ_LEN=9\n",
    "\n",
    "# read in peptide sequences and targets:\n",
    "X_train,y_train = data_io_func.read_pep(\"data/f000\",MAX_PEP_SEQ_LEN)\n",
    "X_val,y_val= data_io_func.read_pep(\"data/c000\",MAX_PEP_SEQ_LEN)\n",
    "\n",
    "# encode data using BLOSUM50:\n",
    "X_train= data_io_func.encode_pep(X_train,MAX_PEP_SEQ_LEN)\n",
    "y_train=np.array(y_train,dtype=theano.config.floatX)\n",
    "X_val= data_io_func.encode_pep(X_val,MAX_PEP_SEQ_LEN)\n",
    "y_val=np.array(y_val,dtype=theano.config.floatX)\n",
    "\n",
    "# data dimensions now:\n",
    "# (N_SEQS, SEQ_LENGTH, N_FEATURES)\n",
    "print(X_train.shape)\n",
    "print(X_val.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we can define the network:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "def build_NN(max_pep_seq_len, n_features, n_hid):\n",
    "\n",
    "    # input layer:\n",
    "    l_in= lasagne.layers.InputLayer((None,max_pep_seq_len,n_features))\n",
    "\n",
    "    # add hidden layer:\n",
    "    l_hid = lasagne.layers.DenseLayer(\n",
    "            l_in,\n",
    "            num_units=n_hid,\n",
    "            nonlinearity=lasagne.nonlinearities.sigmoid,\n",
    "            W=lasagne.init.Normal())\n",
    "\n",
    "    # output layer:\n",
    "    l_out = lasagne.layers.DenseLayer(\n",
    "            l_hid,\n",
    "            num_units=1,\n",
    "            nonlinearity=lasagne.nonlinearities.sigmoid,\n",
    "            W=lasagne.init.Normal())\n",
    "    return l_out,l_in\n",
    "\n",
    "# build a network with 10 hidden neurons:\n",
    "N_FEATURES=21\n",
    "N_HID=10\n",
    "network,inp = build_NN(max_pep_seq_len=MAX_PEP_SEQ_LEN, n_features=N_FEATURES, n_hid=N_HID)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Next we need to define and compile training and validation functions:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "sym_target = T.vector('targets',dtype='float32')\n",
    "sym_l_rate=T.scalar()\n",
    "\n",
    "# TRAINING FUNCTION -----------------------------------------------------------\n",
    "\n",
    "prediction = lasagne.layers.get_output(network)\n",
    "loss = lasagne.objectives.squared_error(prediction.flatten(), sym_target)\n",
    "loss = loss.mean()\n",
    "params = lasagne.layers.get_all_params(network, trainable=True)\n",
    "updates = lasagne.updates.sgd(loss, params, learning_rate=sym_l_rate)\n",
    "\n",
    "# training function:\n",
    "train_fn = theano.function([inp.input_var, sym_target, sym_l_rate], loss, updates=updates)\n",
    "\n",
    "# VALIDATION FUNCTION ----------------------------------------------------------\n",
    "\n",
    "val_prediction = lasagne.layers.get_output(network, deterministic=True)\n",
    "val_loss = lasagne.objectives.squared_error(val_prediction.flatten(),sym_target)\n",
    "val_loss = val_loss.mean()\n",
    "\n",
    "# validation function:\n",
    "val_fn = theano.function([inp.input_var, sym_target], val_loss)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here is a simple training loop:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# Start training loop...\n",
      "Epoch 1\ttraining error: 0.1142\ttime: 0.004 s\n",
      "Epoch 2\ttraining error: 0.1115\ttime: 0.003 s\n",
      "Epoch 3\ttraining error: 0.1103\ttime: 0.002 s\n",
      "Epoch 4\ttraining error: 0.1085\ttime: 0.003 s\n",
      "Epoch 5\ttraining error: 0.1072\ttime: 0.003 s\n",
      "Epoch 6\ttraining error: 0.1051\ttime: 0.003 s\n",
      "Epoch 7\ttraining error: 0.1045\ttime: 0.003 s\n",
      "Epoch 8\ttraining error: 0.1035\ttime: 0.003 s\n",
      "Epoch 9\ttraining error: 0.1026\ttime: 0.003 s\n",
      "Epoch 10\ttraining error: 0.1015\tvalidation error: 0.1016\ttime: 0.004 s\n",
      "Epoch 11\ttraining error: 0.1004\ttime: 0.003 s\n",
      "Epoch 12\ttraining error: 0.0996\ttime: 0.003 s\n",
      "Epoch 13\ttraining error: 0.0985\ttime: 0.003 s\n",
      "Epoch 14\ttraining error: 0.0979\ttime: 0.003 s\n",
      "Epoch 15\ttraining error: 0.0976\ttime: 0.004 s\n",
      "Epoch 16\ttraining error: 0.0966\ttime: 0.004 s\n",
      "Epoch 17\ttraining error: 0.0962\ttime: 0.003 s\n",
      "Epoch 18\ttraining error: 0.0954\ttime: 0.004 s\n",
      "Epoch 19\ttraining error: 0.095\ttime: 0.003 s\n",
      "Epoch 20\ttraining error: 0.0949\tvalidation error: 0.0933\ttime: 0.005 s\n",
      "Epoch 21\ttraining error: 0.0949\ttime: 0.003 s\n",
      "Epoch 22\ttraining error: 0.0938\ttime: 0.003 s\n",
      "Epoch 23\ttraining error: 0.0938\ttime: 0.003 s\n",
      "Epoch 24\ttraining error: 0.0929\ttime: 0.003 s\n",
      "Epoch 25\ttraining error: 0.0929\ttime: 0.003 s\n",
      "Epoch 26\ttraining error: 0.0922\ttime: 0.003 s\n",
      "Epoch 27\ttraining error: 0.0918\ttime: 0.003 s\n",
      "Epoch 28\ttraining error: 0.0916\ttime: 0.003 s\n",
      "Epoch 29\ttraining error: 0.0913\ttime: 0.003 s\n",
      "Epoch 30\ttraining error: 0.0909\tvalidation error: 0.091\ttime: 0.004 s\n",
      "Epoch 31\ttraining error: 0.0906\ttime: 0.003 s\n",
      "Epoch 32\ttraining error: 0.0903\ttime: 0.003 s\n",
      "Epoch 33\ttraining error: 0.0896\ttime: 0.004 s\n",
      "Epoch 34\ttraining error: 0.09\ttime: 0.003 s\n",
      "Epoch 35\ttraining error: 0.089\ttime: 0.003 s\n",
      "Epoch 36\ttraining error: 0.0889\ttime: 0.003 s\n",
      "Epoch 37\ttraining error: 0.0887\ttime: 0.003 s\n",
      "Epoch 38\ttraining error: 0.0877\ttime: 0.003 s\n",
      "Epoch 39\ttraining error: 0.0881\ttime: 0.003 s\n",
      "Epoch 40\ttraining error: 0.0882\tvalidation error: 0.0884\ttime: 0.004 s\n",
      "Epoch 41\ttraining error: 0.0871\ttime: 0.003 s\n",
      "Epoch 42\ttraining error: 0.0868\ttime: 0.003 s\n",
      "Epoch 43\ttraining error: 0.0868\ttime: 0.003 s\n",
      "Epoch 44\ttraining error: 0.0862\ttime: 0.003 s\n",
      "Epoch 45\ttraining error: 0.0857\ttime: 0.003 s\n",
      "Epoch 46\ttraining error: 0.0861\ttime: 0.003 s\n",
      "Epoch 47\ttraining error: 0.0855\ttime: 0.003 s\n",
      "Epoch 48\ttraining error: 0.085\ttime: 0.003 s\n",
      "Epoch 49\ttraining error: 0.0847\ttime: 0.002 s\n",
      "Epoch 50\ttraining error: 0.0842\tvalidation error: 0.0852\ttime: 0.004 s\n",
      "Epoch 51\ttraining error: 0.0844\ttime: 0.003 s\n",
      "Epoch 52\ttraining error: 0.0836\ttime: 0.003 s\n",
      "Epoch 53\ttraining error: 0.0829\ttime: 0.003 s\n",
      "Epoch 54\ttraining error: 0.0835\ttime: 0.003 s\n",
      "Epoch 55\ttraining error: 0.0825\ttime: 0.003 s\n",
      "Epoch 56\ttraining error: 0.0821\ttime: 0.002 s\n",
      "Epoch 57\ttraining error: 0.0816\ttime: 0.002 s\n",
      "Epoch 58\ttraining error: 0.0816\ttime: 0.002 s\n",
      "Epoch 59\ttraining error: 0.0804\ttime: 0.003 s\n",
      "Epoch 60\ttraining error: 0.0809\tvalidation error: 0.0815\ttime: 0.004 s\n",
      "Epoch 61\ttraining error: 0.0806\ttime: 0.003 s\n",
      "Epoch 62\ttraining error: 0.0803\ttime: 0.002 s\n",
      "Epoch 63\ttraining error: 0.0796\ttime: 0.003 s\n",
      "Epoch 64\ttraining error: 0.0781\ttime: 0.003 s\n",
      "Epoch 65\ttraining error: 0.079\ttime: 0.003 s\n",
      "Epoch 66\ttraining error: 0.078\ttime: 0.003 s\n",
      "Epoch 67\ttraining error: 0.0786\ttime: 0.002 s\n",
      "Epoch 68\ttraining error: 0.0778\ttime: 0.002 s\n",
      "Epoch 69\ttraining error: 0.0776\ttime: 0.003 s\n",
      "Epoch 70\ttraining error: 0.077\tvalidation error: 0.0778\ttime: 0.004 s\n",
      "Epoch 71\ttraining error: 0.0769\ttime: 0.003 s\n",
      "Epoch 72\ttraining error: 0.0761\ttime: 0.003 s\n",
      "Epoch 73\ttraining error: 0.0761\ttime: 0.003 s\n",
      "Epoch 74\ttraining error: 0.0758\ttime: 0.003 s\n",
      "Epoch 75\ttraining error: 0.0752\ttime: 0.003 s\n",
      "Epoch 76\ttraining error: 0.0747\ttime: 0.003 s\n",
      "Epoch 77\ttraining error: 0.0748\ttime: 0.002 s\n",
      "Epoch 78\ttraining error: 0.0745\ttime: 0.002 s\n",
      "Epoch 79\ttraining error: 0.0733\ttime: 0.002 s\n",
      "Epoch 80\ttraining error: 0.074\tvalidation error: 0.0744\ttime: 0.004 s\n",
      "Epoch 81\ttraining error: 0.0739\ttime: 0.003 s\n",
      "Epoch 82\ttraining error: 0.0729\ttime: 0.003 s\n",
      "Epoch 83\ttraining error: 0.0734\ttime: 0.002 s\n",
      "Epoch 84\ttraining error: 0.072\ttime: 0.003 s\n",
      "Epoch 85\ttraining error: 0.0715\ttime: 0.003 s\n",
      "Epoch 86\ttraining error: 0.0717\ttime: 0.003 s\n",
      "Epoch 87\ttraining error: 0.0715\ttime: 0.003 s\n",
      "Epoch 88\ttraining error: 0.0707\ttime: 0.003 s\n",
      "Epoch 89\ttraining error: 0.0707\ttime: 0.003 s\n",
      "Epoch 90\ttraining error: 0.071\tvalidation error: 0.0721\ttime: 0.004 s\n",
      "Epoch 91\ttraining error: 0.0703\ttime: 0.003 s\n",
      "Epoch 92\ttraining error: 0.0698\ttime: 0.002 s\n",
      "Epoch 93\ttraining error: 0.0696\ttime: 0.003 s\n",
      "Epoch 94\ttraining error: 0.0691\ttime: 0.003 s\n",
      "Epoch 95\ttraining error: 0.0692\ttime: 0.003 s\n",
      "Epoch 96\ttraining error: 0.0686\ttime: 0.003 s\n",
      "Epoch 97\ttraining error: 0.0688\ttime: 0.002 s\n",
      "Epoch 98\ttraining error: 0.068\ttime: 0.002 s\n",
      "Epoch 99\ttraining error: 0.0681\ttime: 0.002 s\n",
      "Epoch 100\ttraining error: 0.0679\tvalidation error: 0.069\ttime: 0.004 s\n",
      "Epoch 101\ttraining error: 0.0674\ttime: 0.003 s\n",
      "Epoch 102\ttraining error: 0.0668\ttime: 0.003 s\n",
      "Epoch 103\ttraining error: 0.0666\ttime: 0.003 s\n",
      "Epoch 104\ttraining error: 0.067\ttime: 0.003 s\n",
      "Epoch 105\ttraining error: 0.0661\ttime: 0.003 s\n",
      "Epoch 106\ttraining error: 0.0666\ttime: 0.003 s\n",
      "Epoch 107\ttraining error: 0.0658\ttime: 0.003 s\n",
      "Epoch 108\ttraining error: 0.0656\ttime: 0.003 s\n",
      "Epoch 109\ttraining error: 0.0656\ttime: 0.002 s\n",
      "Epoch 110\ttraining error: 0.065\tvalidation error: 0.0665\ttime: 0.004 s\n",
      "Epoch 111\ttraining error: 0.0653\ttime: 0.003 s\n",
      "Epoch 112\ttraining error: 0.0647\ttime: 0.003 s\n",
      "Epoch 113\ttraining error: 0.0642\ttime: 0.002 s\n",
      "Epoch 114\ttraining error: 0.0645\ttime: 0.003 s\n",
      "Epoch 115\ttraining error: 0.064\ttime: 0.003 s\n",
      "Epoch 116\ttraining error: 0.0643\ttime: 0.003 s\n",
      "Epoch 117\ttraining error: 0.0639\ttime: 0.003 s\n",
      "Epoch 118\ttraining error: 0.0636\ttime: 0.002 s\n",
      "Epoch 119\ttraining error: 0.0631\ttime: 0.002 s\n",
      "Epoch 120\ttraining error: 0.0633\tvalidation error: 0.0641\ttime: 0.004 s\n",
      "Epoch 121\ttraining error: 0.0625\ttime: 0.003 s\n",
      "Epoch 122\ttraining error: 0.0627\ttime: 0.003 s\n",
      "Epoch 123\ttraining error: 0.0627\ttime: 0.003 s\n",
      "Epoch 124\ttraining error: 0.0623\ttime: 0.003 s\n",
      "Epoch 125\ttraining error: 0.0621\ttime: 0.003 s\n",
      "Epoch 126\ttraining error: 0.0621\ttime: 0.003 s\n",
      "Epoch 127\ttraining error: 0.062\ttime: 0.003 s\n",
      "Epoch 128\ttraining error: 0.0614\ttime: 0.003 s\n",
      "Epoch 129\ttraining error: 0.0611\ttime: 0.003 s\n",
      "Epoch 130\ttraining error: 0.0607\tvalidation error: 0.0627\ttime: 0.004 s\n",
      "Epoch 131\ttraining error: 0.0611\ttime: 0.003 s\n",
      "Epoch 132\ttraining error: 0.0607\ttime: 0.002 s\n",
      "Epoch 133\ttraining error: 0.0606\ttime: 0.002 s\n",
      "Epoch 134\ttraining error: 0.0605\ttime: 0.003 s\n",
      "Epoch 135\ttraining error: 0.06\ttime: 0.003 s\n",
      "Epoch 136\ttraining error: 0.06\ttime: 0.002 s\n",
      "Epoch 137\ttraining error: 0.0597\ttime: 0.002 s\n",
      "Epoch 138\ttraining error: 0.0599\ttime: 0.002 s\n",
      "Epoch 139\ttraining error: 0.059\ttime: 0.003 s\n",
      "Epoch 140\ttraining error: 0.0588\tvalidation error: 0.0604\ttime: 0.004 s\n",
      "Epoch 141\ttraining error: 0.0588\ttime: 0.003 s\n",
      "Epoch 142\ttraining error: 0.058\ttime: 0.003 s\n",
      "Epoch 143\ttraining error: 0.059\ttime: 0.003 s\n",
      "Epoch 144\ttraining error: 0.0588\ttime: 0.003 s\n",
      "Epoch 145\ttraining error: 0.0584\ttime: 0.003 s\n",
      "Epoch 146\ttraining error: 0.0581\ttime: 0.003 s\n",
      "Epoch 147\ttraining error: 0.0585\ttime: 0.003 s\n",
      "Epoch 148\ttraining error: 0.0581\ttime: 0.002 s\n",
      "Epoch 149\ttraining error: 0.0578\ttime: 0.002 s\n",
      "Epoch 150\ttraining error: 0.0579\tvalidation error: 0.058\ttime: 0.004 s\n",
      "Epoch 151\ttraining error: 0.0575\ttime: 0.003 s\n",
      "Epoch 152\ttraining error: 0.0572\ttime: 0.003 s\n",
      "Epoch 153\ttraining error: 0.057\ttime: 0.002 s\n",
      "Epoch 154\ttraining error: 0.0573\ttime: 0.003 s\n",
      "Epoch 155\ttraining error: 0.0563\ttime: 0.003 s\n",
      "Epoch 156\ttraining error: 0.057\ttime: 0.003 s\n",
      "Epoch 157\ttraining error: 0.0558\ttime: 0.003 s\n",
      "Epoch 158\ttraining error: 0.0566\ttime: 0.003 s\n",
      "Epoch 159\ttraining error: 0.0562\ttime: 0.003 s\n",
      "Epoch 160\ttraining error: 0.056\tvalidation error: 0.0573\ttime: 0.004 s\n",
      "Epoch 161\ttraining error: 0.0553\ttime: 0.003 s\n",
      "Epoch 162\ttraining error: 0.0557\ttime: 0.003 s\n",
      "Epoch 163\ttraining error: 0.0557\ttime: 0.003 s\n",
      "Epoch 164\ttraining error: 0.0551\ttime: 0.003 s\n",
      "Epoch 165\ttraining error: 0.0552\ttime: 0.003 s\n",
      "Epoch 166\ttraining error: 0.0549\ttime: 0.002 s\n",
      "Epoch 167\ttraining error: 0.0549\ttime: 0.002 s\n",
      "Epoch 168\ttraining error: 0.0552\ttime: 0.003 s\n",
      "Epoch 169\ttraining error: 0.0549\ttime: 0.003 s\n",
      "Epoch 170\ttraining error: 0.0545\tvalidation error: 0.0564\ttime: 0.004 s\n",
      "Epoch 171\ttraining error: 0.0543\ttime: 0.003 s\n",
      "Epoch 172\ttraining error: 0.0542\ttime: 0.002 s\n",
      "Epoch 173\ttraining error: 0.0537\ttime: 0.002 s\n",
      "Epoch 174\ttraining error: 0.0537\ttime: 0.003 s\n",
      "Epoch 175\ttraining error: 0.0535\ttime: 0.003 s\n",
      "Epoch 176\ttraining error: 0.0534\ttime: 0.003 s\n",
      "Epoch 177\ttraining error: 0.0532\ttime: 0.003 s\n",
      "Epoch 178\ttraining error: 0.0534\ttime: 0.003 s\n",
      "Epoch 179\ttraining error: 0.0528\ttime: 0.003 s\n",
      "Epoch 180\ttraining error: 0.053\tvalidation error: 0.0544\ttime: 0.004 s\n",
      "Epoch 181\ttraining error: 0.053\ttime: 0.003 s\n",
      "Epoch 182\ttraining error: 0.0528\ttime: 0.002 s\n",
      "Epoch 183\ttraining error: 0.0528\ttime: 0.002 s\n",
      "Epoch 184\ttraining error: 0.053\ttime: 0.003 s\n",
      "Epoch 185\ttraining error: 0.0527\ttime: 0.003 s\n",
      "Epoch 186\ttraining error: 0.0519\ttime: 0.003 s\n",
      "Epoch 187\ttraining error: 0.0524\ttime: 0.003 s\n",
      "Epoch 188\ttraining error: 0.0521\ttime: 0.002 s\n",
      "Epoch 189\ttraining error: 0.052\ttime: 0.002 s\n",
      "Epoch 190\ttraining error: 0.0516\tvalidation error: 0.0535\ttime: 0.004 s\n",
      "Epoch 191\ttraining error: 0.0514\ttime: 0.003 s\n",
      "Epoch 192\ttraining error: 0.0517\ttime: 0.003 s\n",
      "Epoch 193\ttraining error: 0.0518\ttime: 0.003 s\n",
      "Epoch 194\ttraining error: 0.0515\ttime: 0.003 s\n",
      "Epoch 195\ttraining error: 0.0514\ttime: 0.003 s\n",
      "Epoch 196\ttraining error: 0.0518\ttime: 0.003 s\n",
      "Epoch 197\ttraining error: 0.0507\ttime: 0.003 s\n",
      "Epoch 198\ttraining error: 0.0515\ttime: 0.003 s\n",
      "Epoch 199\ttraining error: 0.0508\ttime: 0.003 s\n",
      "# Best epoch: 190\ttrain error: 0.0516\tvalidation error: 0.0535\n",
      "# Time for training: 0.01 min\n",
      "# Done!\n"
     ]
    }
   ],
   "source": [
    "def iterate_minibatches(pep, targets, batchsize):\n",
    "    assert pep.shape[0] == targets.shape[0]\n",
    "    # shuffle:\n",
    "    indices = np.arange(len(pep))\n",
    "    np.random.shuffle(indices)\n",
    "    for start_idx in range(0, len(pep) - batchsize + 1, batchsize):\n",
    "        excerpt = indices[start_idx:start_idx + batchsize]\n",
    "        yield pep[excerpt],targets[excerpt]\n",
    "\n",
    "\n",
    "\n",
    "# TRAINING LOOP ----------------------------------------------------------------\n",
    "\n",
    "EPOCHS=range(1,200)\n",
    "LEARNING_RATE=0.01\n",
    "BATCH_SIZE=200\n",
    "\n",
    "print(\"# Start training loop...\")\n",
    "\n",
    "start_time = time.time()\n",
    "\n",
    "b_epoch=0\n",
    "b_train_err=99\n",
    "b_val_err=99\n",
    "\n",
    "for e in EPOCHS:\n",
    "\n",
    "    train_err = 0\n",
    "    train_batches = 0\n",
    "    val_err = 0\n",
    "    val_batches = 0\n",
    "    e_start_time = time.time()\n",
    "\n",
    "    # shuffle training examples and iterate through minbatches:\n",
    "    for batch in iterate_minibatches(X_train, y_train, BATCH_SIZE):\n",
    "        Xinp, target = batch\n",
    "        train_err += train_fn(Xinp, target, LEARNING_RATE)\n",
    "        train_batches += 1\n",
    "\n",
    "    if e%10 == 0:\n",
    "        # predict validation set:\n",
    "        for batch in iterate_minibatches(X_val, y_val, BATCH_SIZE):\n",
    "            Xinp, target = batch\n",
    "            val_err += val_fn(Xinp, target)\n",
    "            val_batches += 1\n",
    "\n",
    "        \n",
    "        # save only best model:\n",
    "        if (val_err/val_batches) < b_val_err:\n",
    "            np.savez('params.npz', lasagne.layers.get_all_param_values(network))\n",
    "            b_val_err = val_err/val_batches\n",
    "            b_train_err = train_err/train_batches\n",
    "            b_epoch = e\n",
    "        # print performance:\n",
    "        print(\"Epoch \" + str(e) +\n",
    "        \"\\ttraining error: \" + str(round(train_err/train_batches, 4)) +\n",
    "        \"\\tvalidation error: \" + str(round(val_err/val_batches, 4)) +\n",
    "        \"\\ttime: \" + str(round(time.time()-e_start_time, 3)) + \" s\")\n",
    "\n",
    "    else:\n",
    "        # print performance:\n",
    "        print(\"Epoch \" + str(e) +\n",
    "        \"\\ttraining error: \" + str(round(train_err/train_batches, 4)) +\n",
    "        \"\\ttime: \" + str(round(time.time()-e_start_time, 3)) + \" s\")\n",
    "\n",
    "# print best performance:\n",
    "print(\"# Best epoch: \" + str(b_epoch) +\n",
    "        \"\\ttrain error: \" + str(round(b_train_err, 4)) +\n",
    "        \"\\tvalidation error: \" + str(round(b_val_err, 4) ))\n",
    "# report total time used for training:\n",
    "print(\"# Time for training: \" + str(round((time.time()-start_time)/60, 3)) + \" min\" )\n",
    "print(\"# Done!\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}