reproduce_models4.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Dropout_CNN.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tgGheJCNKKMi",
        "outputId": "ec56b274-b7f3-484b-9bb9-24f0df50f129"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive', force_remount=True)\n",
        "import sys\n",
        "#sys.path.append('/content/gdrive/hipsternet-master')"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qCu02wlCPYnd"
      },
      "source": [
        "sys.path.insert(0,'/content/drive/My Drive/hipsternet-master')"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XN3bBYMlN5pZ"
      },
      "source": [
        "import hipsternet"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "a9kb53pqP3zh"
      },
      "source": [
        "import numpy as np\n",
        "import hipsternet.input_data as input_data\n",
        "import hipsternet.neuralnet as nn\n",
        "from hipsternet.solver import *"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RXvo4y6dk_T5"
      },
      "source": [
        "import tensorflow as tf\n",
        "import sympy\n",
        "import numpy as np\n",
        "import seaborn as sns\n",
        "import collections\n",
        "import torch\n",
        "from sklearn.model_selection import GridSearchCV\n",
        "from sklearn.metrics import make_scorer\n",
        "from __future__ import print_function\n",
        "\n",
        "np.random.seed(123)\n",
        "\n",
        "# visualization tools\n",
        "%matplotlib inline\n",
        "import matplotlib.pyplot as plt"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dsz5TNAzlC8t",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "defa88f7-7e50-4eed-9705-83b3448e5768"
      },
      "source": [
        "(x_train, y_train), (x_test, y_test) = tf.keras.datasets.mnist.load_data()"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/mnist.npz\n",
            "11493376/11490434 [==============================] - 0s 0us/step\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 549
        },
        "id": "sBqsQBIoazGU",
        "outputId": "4c9c84a6-a362-498d-9696-5d25d89aaaf0"
      },
      "source": [
        "import tensorflow_datasets as tfds \n",
        "from keras.preprocessing.image import ImageDataGenerator\n",
        "\n",
        "#Generating rotated images: rotated 15%, fill empty pixal with closest color\n",
        "datagen = ImageDataGenerator(rotation_range=15,horizontal_flip=False,fill_mode='nearest')\n",
        "datagen.fit(x_train.reshape(x_train.shape[0],28,28,1))\n",
        "\n",
        "#Print the first rotated image\n",
        "print('Before rotating 15%')\n",
        "plt.imshow(x_train[0])\n",
        "plt.colorbar()\n",
        "plt.show()\n",
        "\n",
        "print('After rotating 15%')\n",
        "for X, Y in datagen.flow(x_train.reshape(x_train.shape[0], 28, 28, 1),y_train.reshape(y_train.shape[0], 1),batch_size=32,shuffle=False):\n",
        "        plt.imshow(X[0].reshape(28,28))\n",
        "        plt.colorbar()\n",
        "        plt.show()\n",
        "        break\n",
        "\n",
        "#append new images to training set: image size should be at least 3D, so reshape is necessary here\n",
        "i=0\n",
        "x_train_aug = x_train\n",
        "y_train_aug = y_train\n",
        "for X, Y in datagen.flow(x_train.reshape(x_train.shape[0], 28, 28, 1),y_train.reshape(y_train.shape[0], 1),batch_size=32,shuffle=False):\n",
        "        x_train_aug = np.vstack([x_train_aug, X.reshape(X.shape[0],28,28)])\n",
        "        y_train_aug = np.append(y_train_aug, Y)\n",
        "        i += 1\n",
        "        if (i > x_train.shape[0]/32): break #fix here"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Before rotating 15%\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAATEAAAD4CAYAAACE9dGgAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAW7ElEQVR4nO3dfbBd1VnH8e/PEIIEtIlpYwppgxhsA7WhXmkcGEoHpZTpTGC0FHTaWNFgS2xR1NKMY1HLDDqFSmtlvEgkzEApLSDRwVKa6fRFS2iIKSSklEiDEC83DSmQvkFy7uMfZ99ybs496+x7z9teN78Ps+ees5/9suYwPKy19lprKyIwM8vVTw26AGZmnXASM7OsOYmZWdacxMwsa05iZpa1I/p5syM1J45ibj9vaXZY+TE/4KV4UZ1c421vnRvP7quVOvahh1+8LyLO7eR+neooiUk6F7gemAX8c0Rckzr+KObyZp3dyS3NLGFTbOz4Gs/uq/Hgfa8pdeysRY8v6PiGHZp2c1LSLOBTwNuBZcDFkpZ1q2BmNhgBjJX8px1JiyV9SdKjkrZL+mCx/ypJuyVtLbbzGs75sKSdkh6T9LZ29+ikJnYasDMinihufDuwEni0g2ua2YAFwYEo15ws4SBwRURskXQs8JCk+4vYxyPiY40HFxWhi4CTgVcDX5R0UkTrAnXSsX8c8FTD96eLfRNIWi1ps6TNB3ixg9uZWb90qyYWESMRsaX4vB/YwSR5osFK4PaIeDEivgPspF5haqnnTycjYjgihiJiaDZzen07M+tQENSi3AYsGK+kFNvqVteVtAQ4FdhU7Foj6WFJ6yTNK/aVqhw16iSJ7QYWN3w/vthnZpkbI0ptwN7xSkqxDU92PUnHAHcCl0fEC8ANwInAcmAEuHa6Ze0kiX0DWCrpBElHUm/HbujgemZWAQHUiFJbGZJmU09gt0bEXQARMRoRtYgYA27k5SbjlCtH005iEXEQWAPcR72de0dEbJ/u9cysOqZQE0uSJOAmYEdEXNewf1HDYRcA24rPG4CLJM2RdAKwFHgwdY+OxolFxL3AvZ1cw8yqJYAD3Vui63Tg3cAjkrYW+9ZSH5K1vLjdLuBSgIjYLukO6qMcDgKXpZ5MQp9H7JtZ9cUUmoptrxXxNWCyGQQtKz8RcTVwddl7OImZ2UQBtYzWSnUSM7MJ6iP28+EkZmaHELVJW4DV5CRmZhPUO/adxMwsU/VxYk5iZpaxMdfEzCxXromZWdYCUcto5XonMTNr4uakmWUrEC/FrEEXozQnMTOboD7Y1c1JM8uYO/bNLFsRohauiZlZxsZcEzOzXNU79vNJDfmU1Mz6wh37Zpa9mseJmVmuPGLfzLI35qeTZpar+gRwJzEzy1QgDnjakZnlKgIPdjWznMmDXc0sX4FrYmaWOXfsm1m2AnlRRDPLV/2VbfmkhnxKamZ94pfnWoXoiPS/4lmvXNDT+z/2p0taxmpHjyXPfe2Je5Lxo9+f/g/tmeuObBnbMvSZ5Ll7az9Ixt/82SuS8V/8kweS8SoLDqMR+5J2AfuBGnAwIoa6USgzG6zDrSb21ojY24XrmFkFROjwqYmZ2cxT79g/fKYdBfAFSQH8U0QMH3qApNXAaoCjOLrD25lZ7+W1xn6nJT0jIt4EvB24TNKZhx4QEcMRMRQRQ7OZ0+HtzKzX6h37KrW1I2mxpC9JelTSdkkfLPbPl3S/pMeLv/OK/ZL0CUk7JT0s6U3t7tFREouI3cXfPcDdwGmdXM/MqqHGT5XaSjgIXBERy4AV1Cs7y4ArgY0RsRTYWHyHeoVoabGtBm5od4NpJzFJcyUdO/4ZOAfYNt3rmVk1jI/Y70ZNLCJGImJL8Xk/sAM4DlgJrC8OWw+cX3xeCdwSdQ8Ar5C0KHWPTvrEFgJ3Sxq/zm0R8fkOrjdjzXr90mQ85sxOxv/vLa9Ixn+0ovWYpvk/mx7v9NU3psdLDdJ//PDYZPxv/+HcZHzTG25rGfvOgR8lz71m9DeS8Vd/NZLx3E3hRSELJG1u+D48Wd84gKQlwKnAJmBhRIwUoWeo5xOoJ7inGk57utg3QgvTTmIR8QTwxumeb2bVFAEHxkonsb1lxodKOga4E7g8Il4oKj/F/SKKh4PT4iEWZjZBvTnZvaeTkmZTT2C3RsRdxe5RSYsiYqRoLo5Pz9gNLG44/fhiX0v5PEc1s76pFfMn223tqF7lugnYERHXNYQ2AKuKz6uAexr2v6d4SrkCeL6h2Tkp18TMbILxIRZdcjrwbuARSVuLfWuBa4A7JF0CPAlcWMTuBc4DdgI/BN7b7gZOYmZ2iO41JyPia9Cyynb2JMcHcNlU7uEkZmZNvMb+YaZ2VnpQ8XU3fyoZP2l26yVjZrIDUUvG//KTv5uMH/GD9AOtX/vsmpaxY3cfTJ47Z296CMbRmzcl4zmrP508fOZOmtkM4+WpzSx7bk6aWba6/HSy55zEzKyJF0U0s2xFiINOYmaWMzcnzSxb7hM7DM157P+S8Yd+vDgZP2n2aDeL01VXjKxIxp/4fvqVbzef+LmWsefH0uO8Fn7iv5LxXprZC+205yRmZtnyODEzy57HiZlZtiLgYPlFEQfOSczMmrg5aWbZcp+YmWUvnMTMLGfu2D/MHBx5Jhn/5N++Mxm/+tz0a9VmPXxMMv7N938yGU/56N5fTsZ3/vrRyXjtueTy5/z2r72/ZWzXB5KncgLfTB9gPRHhPjEzy5qo+emkmeXMfWJmli3PnTSzvEW9XywXTmJm1sRPJ80sW+GOfTPLnZuTNsH8f/l6Mv7Kf/u5ZLz27L5k/ORTfq9lbPuZ65Lnbhh+SzL+quc6W9NLX2891uuE9M9iA5TT08m2dUZJ6yTtkbStYd98SfdLerz4O6+3xTSzfomoJ7EyWxWUafjeDJx7yL4rgY0RsRTYWHw3sxliLFRqq4K2SSwivgIc2p5ZCawvPq8Hzu9yucxsgCLKbVUw3T6xhRExPmnuGWBhqwMlrQZWAxxFeh6emQ1eIMYyejrZcUkjIki8VyEihiNiKCKGZjOn09uZWR9Eya0KppvERiUtAij+7ulekcxsoGZgx/5kNgCris+rgHu6Uxwzq4SMqmJt+8QkfRo4C1gg6WngI8A1wB2SLgGeBC7sZSFnutreZzs6/8ALR0773JN/59Fk/Ls3zEpfYKw27XtbdVWlllVG2yQWERe3CJ3d5bKYWQUEMDbWnSQmaR3wDmBPRJxS7LsK+APgu8VhayPi3iL2YeASoAZ8ICLua3ePfB5BmFl/BBAqt7V3M83jTAE+HhHLi208gS0DLgJOLs75R0ltmgJOYmY2iW6NE2sxzrSVlcDtEfFiRHwH2Amc1u4kJzEza1a+Y3+BpM0N2+qSd1gj6eFiWuP4tMXjgKcajnm62JfkCeBmdogpDZ/YGxFDU7zBDcDfUE+DfwNcC7RexaAN18TMrFkPh1hExGhE1CJiDLiRl5uMu4HFDYceX+xLck1sBnj9h77dMvbeN6QfIv/Lazcm429552XJ+LGfeSAZtwwFRJeeTk5G0qKGaYsXAOMr5GwAbpN0HfBqYCnwYLvrOYmZ2SS6NsRisnGmZ0laTr0utwu4FCAitku6A3gUOAhcFhFtByI6iZlZsy6Nxm8xzvSmxPFXA1dP5R5OYmbWrCJTispwEjOzicYHu2bCSczMmlRlwcMynMTMrFkPn052m5OYmTWRa2LWT7Xnnm8Ze/Z9r0+e+78bfpSMX/nRW5LxD194QTIe//2zLWOLr27zzrac2jQzSYXWCivDSczMDlF6hYpKcBIzs2auiZlZ1sYGXYDynMTMbCKPEzOz3PnppJnlLaMk5vXEzCxrronNcGPf3JGMX/RXf5aM3/qRjyXjW1ekx5GxonXo5LlrkqcuvXEkGT/4xK70vW3a3Jw0s3wFnnZkZplzTczMcubmpJnlzUnMzLLmJGZmuVK4OWlmufPTScvF/HXpNb3WPJZ+7+TPXPN0Mv7pX7ivZWz7e/4hee7rFv9+Mv5Lf5Ueq117/Ilk3FrLqSbWdsS+pHWS9kja1rDvKkm7JW0ttvN6W0wz66sevgG828pMO7oZOHeS/R+PiOXFdm93i2VmAxMv94u126qgbRKLiK8A+/pQFjOrihlWE2tljaSHi+bmvFYHSVotabOkzQd4sYPbmVm/aKzcVgXTTWI3ACcCy4ER4NpWB0bEcEQMRcTQbOZM83ZmZpObVhKLiNGIqEXEGHAjcFp3i2VmAzXTm5OSFjV8vQDY1upYM8tMZh37bceJSfo0cBawQNLTwEeAsyQtp56LdwGX9rCMNkD6z63J+A9/61XJ+K++649axjZ96Prkud966z8n47+z5Jxk/PkzkmFLqUiCKqNtEouIiyfZfVMPymJmVTGTkpiZHV5EdZ48luEkZmYTVai/qwy/KMTMmnXp6WSLaYvzJd0v6fHi77xivyR9QtLOYgzqm8oU1UnMzJp1b4jFzTRPW7wS2BgRS4GNxXeAtwNLi2019fGobTmJmVmTbg2xaDFtcSWwvvi8Hji/Yf8tUfcA8IpDhnNNyn1i1pHa6J5kfOEnWsd//OcHk+cerSOT8RuX/Hsy/o4LLm997bs3Jc897PW2T2xhRIy/j+8ZYGHx+TjgqYbjni72Jd/d5yRmZhPFlJ5OLpC0ueH7cEQMl75VREidPUZwEjOzZuXTyt6IGJri1UclLYqIkaK5OF5d3w0sbjju+GJfkvvEzKxJj6cdbQBWFZ9XAfc07H9P8ZRyBfB8Q7OzJdfEzKxZl/rEWkxbvAa4Q9IlwJPAhcXh9wLnATuBHwLvLXMPJzEzm6iLK1S0mLYIcPYkxwaQfqnDJJzEzGwCkdeIfScxM2viJGYzxtgZy5Px/3nnUcn4Kct3tYy1GwfWzif3nZqMH33P5mTcEpzEzCxrTmJmlq3MVrFwEjOzZk5iZpYzL4poZllzc9LM8lWh17GV4SRmZs2cxKwqNHRKMv7tD7RZs+v09cn4mUe9NOUylfViHEjGH9h3QvoCY23nDtskPGLfzLKnsXyymJOYmU3kPjEzy52bk2aWNycxM8uZa2JmljcnMTPL1tTedjRwTmIZOOKE1ybj//PeV7eMXfWu25Pn/uYxe6dVpm5YO5p+Sc6Xr1+RjM9b//VuFscKuY0Ta/u2I0mLJX1J0qOStkv6YLF/vqT7JT1e/J3X++KaWV9ElNsqoMwr2w4CV0TEMmAFcJmkZcCVwMaIWApsLL6b2QzQ41e2dVXbJBYRIxGxpfi8H9hB/dXiK4HxOSnrgfN7VUgz66OYwlYBU+oTk7QEOBXYBCxseLHlM8DCFuesBlYDHMXR0y2nmfXRjOzYl3QMcCdweUS8IOknsYgIafLKZUQMA8MAP6P5FcndZpaSUxIr0yeGpNnUE9itEXFXsXtU0qIivgjY05simllfBVl17Letiale5boJ2BER1zWENgCrqL+SfBVwT09KOAMcseQ1yfjzv7IoGX/XX38+Gf/DV9yVjPfSFSPpYRBf/8fWwyjm3/xg8tx5Yx5CMShV6bQvo0xz8nTg3cAjkrYW+9ZST153SLoEeBK4sDdFNLO+m0lJLCK+Rn3822TO7m5xzGzQchvs6hH7ZjZRhBdFNLPM5ZPDnMTMrJmbk2aWrwDcnDSzrOWTw5zEyjpi0c+3jO1bNzd57vtO+HIyfvGxo9MqUzes2X1GMr7lhuXJ+ILPbUvG5+/3WK8cuTlpZlnr5tNJSbuA/UANOBgRQ5LmA58BlgC7gAsj4nvTuX6paUdmdhjpzSoWb42I5RExPoWja0t5OYmZ2QT1wa5RautA15bychIzs2ZjJTdYIGlzw7Z6kqsF8AVJDzXESy3lVYb7xMysyRRqWXsbmoitnBERuyW9Crhf0rcag6mlvMpwTczMJupyn1hE7C7+7gHuBk6ji0t5OYmZ2SHqcyfLbO1Imivp2PHPwDnANl5eygs6XMrrsGlOvvS2dI33pT/el4yv/cV7W8bO+ekfTKtM3TJa+1HL2Jkbrkie+7q/+FYyPv+59DivjBYAtano3oKHC4G7i5WgjwBui4jPS/oGXVrK67BJYmZWUhdfnhsRTwBvnGT/s3RpKS8nMTNrVpGlp8twEjOzZvnkMCcxM2umsXx6O53EzGyiIKsnNk5iZjaB6HhKUV85iZlZMyex6tl1fnpc77ff8Nme3ftTz52YjF//5XOScdVavWyq7nUf/U7L2NLRTclza8moHbacxMwsW+4TM7Pc+emkmWUs3Jw0s4wFTmJmlrl8WpNOYmbWzOPEzCxvMymJSVoM3EJ9XaAAhiPieklXAX8AfLc4dG1EtF50a8BOet+Dyfg73vcrfSpJs5NIl60dj/WyroqAWj7tyTI1sYPAFRGxpVih8SFJ9xexj0fEx3pXPDMbiJlUEyveSDJSfN4vaQdwXK8LZmYDlFESm9Ia+5KWAKcC43NZ1kh6WNI6SfNanLN6/HVOB3ixo8KaWR8EMBbltgooncQkHQPcCVweES8ANwAnAsup19Suney8iBiOiKGIGJrNnC4U2cx6KyDGym0VUOrppKTZ1BPYrRFxF0BEjDbEbwT+vSclNLP+CrLq2G9bE1P9NSU3ATsi4rqG/YsaDruA+muYzGwmiCi3VUCZmtjpwLuBRyRtLfatBS6WtJx63t4FXNqTEppZ/1UkQZVR5unk14DJFrSq7JgwM+tEdWpZZXjEvplNFICX4jGzrLkmZmb5mnnTjszscBIQFRkDVoaTmJk1q8ho/DKcxMysmfvEzCxbEX46aWaZc03MzPIVRC2fpTadxMxsovGleDLhJGZmzTIaYjGlRRHNbOYLIMai1FaGpHMlPSZpp6Qru11eJzEzmyi6tyiipFnAp4C3A8uor36zrJvFdXPSzJp0sWP/NGBnRDwBIOl2YCXwaLdu0Ncktp/v7f1ifO7Jhl0LgL39LMMUVLVsVS0XuGzT1c2yvbbTC+zne/d9MT63oOThR0na3PB9OCKGG74fBzzV8P1p4M2dlrFRX5NYRLyy8bukzREx1M8ylFXVslW1XOCyTVfVyhYR5w66DFPhPjEz66XdwOKG78cX+7rGSczMeukbwFJJJ0g6ErgI2NDNGwy6Y3+4/SEDU9WyVbVc4LJNV5XL1pGIOChpDXAfMAtYFxHbu3kPRUZzpMzMDuXmpJllzUnMzLI2kCTW62kInZC0S9IjkrYeMv5lEGVZJ2mPpG0N++ZLul/S48XfeRUq21WSdhe/3VZJ5w2obIslfUnSo5K2S/pgsX+gv12iXJX43XLV9z6xYhrCt4HfoD7w7RvAxRHRtRG8nZC0CxiKiIEPjJR0JvB94JaIOKXY93fAvoi4pvgfwLyI+FBFynYV8P2I+Fi/y3NI2RYBiyJii6RjgYeA84HfZYC/XaJcF1KB3y1Xg6iJ/WQaQkS8BIxPQ7BDRMRXgH2H7F4JrC8+r6f+H0HftShbJUTESERsKT7vB3ZQHzk+0N8uUS7rwCCS2GTTEKr0LzKAL0h6SNLqQRdmEgsjYqT4/AywcJCFmcQaSQ8Xzc2BNHUbSVoCnApsokK/3SHlgor9bjlxx36zMyLiTdRn3V9WNJsqKep9AVUaI3MDcCKwHBgBrh1kYSQdA9wJXB4RLzTGBvnbTVKuSv1uuRlEEuv5NIRORMTu4u8e4G7qzd8qGS36Vsb7WPYMuDw/ERGjEVGL+ksLb2SAv52k2dQTxa0RcVexe+C/3WTlqtLvlqNBJLGeT0OYLklziw5XJM0FzgG2pc/quw3AquLzKuCeAZZlgvEEUbiAAf12kgTcBOyIiOsaQgP97VqVqyq/W64GMmK/eIT897w8DeHqvhdiEpJ+gXrtC+pTsm4bZNkkfRo4i/pSLaPAR4B/Be4AXgM8CVwYEX3vYG9RtrOoN4kC2AVc2tAH1c+ynQF8FXgEGF+5by31/qeB/XaJcl1MBX63XHnakZllzR37ZpY1JzEzy5qTmJllzUnMzLLmJGZmWXMSM7OsOYmZWdb+H71dPbjcA328AAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        },
        {
          "output_type": "stream",
          "text": [
            "After rotating 15%\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAATEAAAD6CAYAAADJPXCrAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAYUUlEQVR4nO3dbZAd1X3n8e+P0SBhSQ4CxfIgDQiDYq/sxIJMgMQswWFtY9a1gq0ygaSwksURW4EqU3FSIbyI8QuqKMfgtcsOtcJoEVXYmApgtF5iDLID4cEYCfOgB7zIWDISIylCBI3ASJqZf17cVnLn4Z7bM/fO3Huufh+qa/qef/ftox7pz+nTp08rIjAzy9Uxra6AmVkjnMTMLGtOYmaWNScxM8uak5iZZc1JzMyy5iRmZlNGUq+kH0naLGmTpM8V5TdI2inpuWK5qGqfv5G0VdLPJH2i7jGmc5zYsZoZs5g9bcczO9q8w1scioNq5Ds+8dHZ8fq+oVLbbnjh4EMRcWGtuKQeoCcinpU0F9gAXAxcChyIiC+P2n4p8G3gLOAk4BHgNyKiZoVmlKpp7QpeCHwV6AK+GRE3pbafxWzO1gWNHNLMEp6OdQ1/x+v7hvjJQyeX2rar5+X5qXhE9AP9xfqApC3AwsQuy4G7I+Ig8AtJW6kktKdq7TDpy0lJXcA3gE8CS4HLiyxqZhkLYLjkfxMhaTFwBvB0UXSNpBckrZY0ryhbCLxatdsO0kmvoT6xs4CtEfFKRBwC7qaSRc0sY0FwOIZKLcB8SeurlpXjfaekOcC9wLURsR+4FTgNWEalpXbzZOvbyOXkeBnz7NEbFX+olQCzeFcDhzOz6TKBVtbeiOhLbSCpm0oCuysi7gOIiN1V8duA7xUfdwK9VbsvKspqmvK7kxGxKiL6IqKvm5lTfTgza1AQDEW5pR5JAm4HtkTELVXlPVWbXQJsLNbXApdJminpVGAJ8JPUMRppiU04Y5pZHoZp2qiFjwBXAC9Keq4ou55KH/oyKl1w24CrACJik6R7gM3AIHB16s4kNJbEngGWFNlyJ3AZ8EcNfJ+ZtYEAhpqUxCLicWC8IR8PJva5Ebix7DEmncQiYlDSNcBDVIZYrI6ITZP9PjNrH01siU25hsaJRcSDJDKqmeUngMMZTZbaUBIzs84TRNMuJ6eDk5iZjRQwlE8OcxIzs5EqI/bz4SRmZqOIoXFvKLYnJzEzG6HSse8kZmaZqowTcxIzs4wNuyVmZrlyS8zMshaIoYxmrncSM7MxfDlpZtkKxKHoanU1SnMSM7MRKoNdfTlpZhlzx76ZZStCDIVbYmaWsWG3xMwsV5WO/XxSQz41NbNp4Y59M8vekMeJmVmuPGLfzLI37LuTZparygPgTmLWKVSnbySjt+JYOYE47MeOzCxXEXiwq5nlTB7samb5CtwSM7PMuWPfzLIVyJMimlm+Kq9syyc15FNTM5smR9HLcyVtAwaAIWAwIvqaUamOU2es1Yz3Lkjv35UeszPUc0Lt2LvSv+KuA4eS8ehOH3vG9j3JeGoc2S8/c1py12PSVWPwuHT8lFUv1Ywd+M+np797VrpPaM4vf5WMd734SjI+PDCQjLdScPSN2P9oROxtwveYWZs4alpiZtZ5IpRVS6zRmgbwA0kbJK1sRoXMrLUqHftdpZZ6JPVK+pGkzZI2SfpcUX6CpIclvVz8nFeUS9LXJG2V9IKkM+sdo9Ekdm5EnAl8Erha0nnj/CFWSlovaf1hDjZ4ODObepU59sssJQwCn4+IpcA5VPLEUuA6YF1ELAHWFZ+hkkuWFMtK4NZ6B2goiUXEzuLnHuB+4KxxtlkVEX0R0dfNzEYOZ2bToNKxr1JL3e+K6I+IZ4v1AWALsBBYDqwpNlsDXFysLwfujIofA8dL6kkdY9JJTNJsSXOPrAMfBzZO9vvMrH0McUypZSIkLQbOAJ4GFkREfxHaBRy5Rb8QeLVqtx1FWU2NdOwvAO5XZfjADOBbEfH9Br7PzNrABEfsz5e0vurzqohYNXojSXOAe4FrI2K/qoYdRURImvScTpNOYhHxCvDhye7fSY6ZNSsZ/9Uf/GYyvuPKt5Pxgwe7k/G5s9+pGVty4mvJfd8z80Ay/vWFTyfjQzGcjD9xsPb/ret1DL96+MRk/CcD70vG/6mrdp/wH376n5L7Xjf/+WT8qXfSXSP/866rkvFT/vapZLzVJvCikL31xodK6qaSwO6KiPuK4t2SeiKiv7hcPDLgcCfQW7X7oqKspnzuo5rZtIiAw8PHlFrqUaXJdTuwJSJuqQqtBVYU6yuAB6rKP1PcpTwHeLPqsnNcHidmZiNULieb1r75CHAF8KKk54qy64GbgHskXQlsBy4tYg8CFwFbgbeBP613ACcxMxujWSP2I+JxqPllF4yzfQBXT+QYTmJmNsKRIRa5cBIzs1HyeuzISczMxvAc+zbCWwvSp/muM1cn48tmtu5Jh3pDKLqU/j/2ecnRJ0PJfdfxejL+xRf/WzJO7+GaoS0H3pvc9Y/3n5SM/3RbbzJ+6g/zfcSucnfSr2wzs0x5emozy54vJ80sW747aWbZ891JM8tWhBh0EjOznPly0syy5T6xo9DwwfSYoONfqT1VDsDl6z+bjH/s1NqvHgM4c872mrE/eXf6lWoHo/ZYKoDH30lPM3Tvvt9Jxk8/rvbxrzx+U3LfL237dDL+/m+kX5s2PLP2X+/Xu05J7lvvVXYfeCM9hm1w2y+T8XbnJGZm2fI4MTPLnseJmVm2ImCwxISH7cJJzMzG8OWkmWXLfWJmlr1wEjOznLlj/2gT6VfmdT//SjJ+0q2nJ+PPLPjtZPx759Z+NdnvfeqWmjGAlw79ejL+V3evSMZP+7v0WK+fLz23ZuzvP/3x5L7H7kt3LvdueDIZT/0zrPdPtN5LEAfrxHMW4T4xM8uaGPLdSTPLmfvEzCxbfnbSzPIWdbt524qTmJmN4buTZpatcMe+meWuoy4nJa0GPgXsiYgPFWUnAN8BFgPbgEsj4o2pq2behv71zWS8+7Hnk/Ffq/P9h2fXntPr/vOWJff9x/4PJuOnfD89Z9fQ/v3JuNZvrhl7/670ux05Jt0a6OSxWq2W093JMm3GO4ALR5VdB6yLiCXAuuKzmXWAiEoSK7O0g7pJLCIeA/aNKl4OrCnW1wAXN7leZtZCw6FSSzuYbJ/YgojoL9Z3AQuaVB8zawMd1SdWT0SEpJp/ZEkrgZUAs3hXo4czsykWiOGM7k5Otqa7JfUAFD9rvg0iIlZFRF9E9HUzc5KHM7PpFCWXdjDZJLYWODK9wQrggeZUx8xartM69iV9G3gKeL+kHZKuBG4CPibpZeC/FJ/NrFM0qSkmabWkPZI2VpXdIGmnpOeK5aKq2N9I2irpZ5I+UaaqdfvEIuLyGqELyhzA6ovBxkY8zd41VDP20lvvTe57Re+Pk/HbFqdvPP/aE8lw8s+W+7sZO1kTW1l3AF8H7hxV/pWI+HJ1gaSlwGXAB4GTgEck/UZE1P4LzuQvJ82sQwUwPKxSS93vGn+IVi3Lgbsj4mBE/ALYCpxVbycnMTMbKYBQuWXyrpH0QnG5Oa8oWwi8WrXNjqIsyUnMzMaIKLcA8yWtr1pWlvj6W4HTgGVAP3BzI3X1A+BmNlb58RN7I6JvQl8dsfvIuqTbgO8VH3cCvVWbLirKktwSM7NRyg2vmGzn/5ExpoVLgCN3LtcCl0maKelUYAnwk3rf55aYmY3VpJGsxRCt86lcdu4AvgCcL2lZcZRtwFUAEbFJ0j3AZiqTlFxd784kOIl1hOMerj2Vz5Nn136dG8Bn//jRZHzvGeljn7gu/djs4K7dybi1oYAoceex1FeNP0Tr9sT2NwI3TuQYTmJmNo72GI1fhpOYmY3VLg9GluAkZmZjOYmZWbaODHbNhJOYmY1xVE2KaGYdqEl3J6eDk5iZjVF7rub24yTWAeLgwZqx07+5I7nvrRf8QTL+vy+5LRn/yyWfTsYHHz+tZqz3//wsue/Q3teTcZsi7TRtawlOYmY2SsMzVEwrJzEzG8stMTPL2nCrK1Cek5iZjeRxYmaWO9+dNLO8ZZTEPCmimWXNLbEON7j91WR8281nJ+MDN61Pxv/5t+9Ixjf/VlfN2GeHP5fct/c725PxwZ2vJeNZPTvTZnw5aWb5CvzYkZllzi0xM8uZLyfNLG9OYmaWNScxM8uVwpeTZpa7Tro7KWk18ClgT0R8qCi7Afgz4F+Kza6PiAenqpI2dWbfnx4H9vV9lybj/++m9Jxgt/U+UTN2zWe/m9z3a90XJ+Mnf/OdZNzzkU1eTi2xMiP27wAuHKf8KxGxrFicwMw6SZRc2kDdllhEPCZp8dRXxczaQmZ9Yo08O3mNpBckrZY0r2k1MrPWy6glNtkkditwGrAM6AdurrWhpJWS1ktaf5jac8GbWfvQcLmlHUwqiUXE7ogYiohh4DbgrMS2qyKiLyL6upk52XqamY1rUklMUk/Vx0uAjc2pjpm1hYwuJ8sMsfg2cD4wX9IO4AvA+ZKWUfljbAOumsI6mtl0yqxjv8zdycvHKb59CupirTA8lAx3Pfp8Mv7qn38gGT/1qqU1Y5su+kZy352XP5qM33/495PxnpufTMYtoZOSmJkdhZzEzCxXon3uPJbhOfbNbKT4j4fA6y31FONI90jaWFV2gqSHJb1c/JxXlEvS1yRtLcagnlmmuk5iZjZW8+5O3sHYxxavA9ZFxBJgXfEZ4JPAkmJZSWU8al1OYmY2VpOSWEQ8BuwbVbwcWFOsrwEuriq/Myp+DBw/ajjXuJzEzGyMZl1O1rAgIvqL9V3AgmJ9IVD9eq4dRVmSO/aPdmps3qj46UvJ+IJHf6dmrPu/1n6dG8BfnJieJujJi96XjM+4Z1HN2OCrO5L7HvXKJ6j5kqp/UasiYlXpw0SE1NioNCcxMxspJnR3cm9E9E3wCLsl9UREf3G5uKco3wn0Vm23qChL8uWkmY01tY8drQVWFOsrgAeqyj9T3KU8B3iz6rKzJrfEzGyMZj12VOOxxZuAeyRdCWwHjkwf/CBwEbAVeBv40zLHcBIzs7GalMRqPLYIcME42wZw9USP4SRmZiO10QwVZTiJmdkIosNmsTCzo4+TmE0rzaw9Y27X/BOT+w71nJCM7/3w3GT89bMHk/Hf/WDtV7p1Kz1O7O2h9DRBuwfmJOO93b9Kxi3BSczMsuYkZmbZ6rSZXc3sKOQkZmY5y2lSRCcxMxvDl5Nmli8PdjWz7DmJ2UQcM2tWMq5Te5PxNz9Ue6zX7nPS84VdeN5Pk/FVC9Yl43OVngjlmOR8Zccl933+UHqM2+Fn5yXjg69sScZtfB6xb2bZ03A+WcxJzMxGcp+YmeXOl5NmljcnMTPLmVtiZpY3JzEzy9bE3nbUcnWTmKRe4E4qL7gMKu+V+6qkE4DvAIuBbcClEfHG1FW1fXW9+93pDRa9NxnuP39+Mv7m2e8k43/04Sdrxq4+4ankvj0z0nNyQb142hPv1P7XcO3mS2vGAN5+In1eTl43MKk6WVpu48TKvLJtEPh8RCwFzgGulrQUuA5YFxFLgHXFZzPrBBHlljZQN4lFRH9EPFusDwBbqLxafDmwpthsDXDxVFXSzKaXotzSDibUJyZpMXAG8DSwoOrFlruoXG6aWe46dbCrpDnAvcC1EbFfVc/ERURI4+dlSSuBlQCzeFdjtTWzaZFTx36ZPjEkdVNJYHdFxH1F8W5JPUW8B9gz3r4RsSoi+iKir5vaL7Qws/ah4XJLO6ibxFRpct0ObImIW6pCa4EVxfoK4IHmV8/Mpl2QVcd+mcvJjwBXAC9Keq4oux64CbhH0pXAdiB9v3yKpV5bBqDklDCg49LTwmz78/9UMzb0WweS+/bM25+MX7Uonf//cO7Lyfi8rtqX6XuH0n/uLYfeTsYffXtJMv6lH34qGV/4w9qx+T9Pnxf9YmMyPrQ/vb9NXrt02pdRN4lFxONUho6M54LmVsfM2kInJTEzO7rkNtjVSczMRorwpIhmlrl8cpiTmJmN5ctJM8tXAL6cNLOs5ZPDOieJHTz/N5Px1/7HoWT8sg9sSMb/du4jNWOLZvwque+CrvQYtG51JeMHozsZ/+5btafL+buf//fkvgf+MT1N0KL/+1oy/oF9LyXjw2/VPjfDh9O/E2udZl5OStoGDABDwGBE9DVzKq9Sjx2Z2dFFw1FqmYCPRsSyiOgrPjdtKi8nMTMbKSawTF7TpvJyEjOzESqDXaPUUlIAP5C0oZjVBpo4lVfH9ImZWROVn6FivqT1VZ9XRcSqUducGxE7Jb0HeFjSiI7U1FReZTiJmdkYE2hl7a3q5xpXROwsfu6RdD9wFsVUXhHRn5rKqwxfTprZSE3sE5M0W9LcI+vAx4GNNHEqL7fEzGyUpj47uQC4v5gKawbwrYj4vqRnaNJUXh2TxHadc2wy/t2zv5aMn96dno8sNZbrB2/PS+57bf/vJ+Nzuw8m409uPzUZP3ZD7XFiJ9+1LbnvnNfSr3QbbJOJ72yaNen3HhGvAB8ep/x1mjSVV8ckMTNrkk57ea6ZHYUyaoE7iZnZWPnkMCcxMxtLw/lcTzqJmdlIwUQGu7ack5iZjSAm9EhRyzmJmdlYTmLT7+QvPpmMX/vF30vGZyw8KRk/vPg9NWOHjk/P99X1Trptvvvd6V/D+x7ZnIwPDwzUjA0m9zSrwUnMzLLlPjEzy53vTppZxsKXk2aWscBJzMwyl8/VpJOYmY3lcWJmlrdOSmKSeoE7qUxuFlTm0P6qpBuAPwP+pdj0+oh4cKoqOtUGd6bfr6hEPD0TWX31fgkZteytE0TAUD5/68q0xAaBz0fEs8U0sxskPVzEvhIRX5666plZS3RSS6x4rVJ/sT4gaQuwcKorZmYtlFESm9CLQiQtBs4Ani6KrpH0gqTVksado1nSSknrJa0/THoaZjNrAwEMR7mlDZROYpLmAPcC10bEfuBW4DRgGZWW2s3j7RcRqyKiLyL6uhvuPTKzqRcQw+WWNlDq7qSkbioJ7K6IuA8gInZXxW8DvjclNTSz6RVk1bFftyWmyruWbge2RMQtVeU9VZtdQuVdcmbWCSLKLW2gTEvsI8AVwIuSnivKrgcul7SMSt7eBlw1JTU0s+nXJgmqjDJ3Jx8HNE4o2zFhZpbSPq2sMjxi38xGCsBT8ZhZ1twSM7N8dd5jR2Z2NAmINhkDVoaTmJmN1Saj8ctwEjOzsdwnZmbZivDdSTPLnFtiZpavIIaGWl2J0pzEzGykI1PxZGJC84mZ2VGiiVPxSLpQ0s8kbZV0XbOr6paYmY0QQDSpJSapC/gG8DFgB/CMpLURsbkpB8AtMTMbLZo6KeJZwNaIeCUiDgF3A8ubWV23xMxsjCZ27C8EXq36vAM4u1lfDtOcxAZ4Y+8j8Q/bq4rmA3unsw4T0K51a9d6ges2Wc2s2ymNfsEAbzz0SPzD/JKbz5K0vurzqohY1WgdJmJak1hE/Hr1Z0nrI6JvOutQVrvWrV3rBa7bZLVb3SLiwiZ+3U6gt+rzoqKsadwnZmZT6RlgiaRTJR0LXAasbeYB3CdmZlMmIgYlXQM8BHQBqyNiUzOP0eokNq3XzhPUrnVr13qB6zZZ7Vy3hkXEg0zhdPaKjJ6RMjMbzX1iZpa1liSxqX4MoRGStkl6UdJzo24dt6IuqyXtkbSxquwESQ9Lern4Oa+N6naDpJ3FuXtO0kUtqluvpB9J2ixpk6TPFeUtPXeJerXFecvVtF9OFo8h/H+qHkMALm/mYwiNkLQN6IuIlo8pknQecAC4MyI+VJR9CdgXETcV/wOYFxF/3SZ1uwE4EBFfnu76jKpbD9ATEc9KmgtsAC4G/oQWnrtEvS6lDc5brlrREpvyxxA6RUQ8BuwbVbwcWFOsr6Hyj2Da1ahbW4iI/oh4tlgfALZQGTne0nOXqJc1oBVJbLzHENrpFxnADyRtkLSy1ZUZx4KI6C/WdwELWlmZcVwj6YXicrMll7rVJC0GzgCepo3O3ah6QZudt5y4Y3+scyPiTOCTwNXFZVNbikpfQDvdXr4VOA1YBvQDN7eyMpLmAPcC10bE/upYK8/dOPVqq/OWm1YksSl/DKEREbGz+LkHuJ/K5W872V30rRzpY9nT4vr8u4jYHRFDUXnf12208NxJ6qaSKO6KiPuK4pafu/Hq1U7nLUetSGJT/hjCZEmaXXS4Imk28HFgY3qvabcWWFGsrwAeaGFdRjiSIAqX0KJzJ0nA7cCWiLilKtTSc1erXu1y3nLVksGuxS3k/8V/PIZw47RXYhyS3kel9QWVpxm+1cq6Sfo2cD6VWQ52A18AvgvcA5wMbAcujYhp72CvUbfzqVwSBbANuKqqD2o663Yu8M/Ai8CRSa+up9L/1LJzl6jX5bTBecuVR+ybWdbcsW9mWXMSM7OsOYmZWdacxMwsa05iZpY1JzEzy5qTmJllzUnMzLL2b3H4c4GVGcTSAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0W3D61sza61Z",
        "outputId": "39d16a24-379e-4054-8d4a-63a5f0a8ad33"
      },
      "source": [
        "#Augmented training sets: x_train_aug, y_train_aug\n",
        "#vectorize data into 2-d np array\n",
        "x_train, x_test = x_train.reshape(x_train.shape[0], x_train.shape[1]*x_train.shape[2]), x_test.reshape(x_test.shape[0], x_test.shape[1] * x_test.shape[2])\n",
        "\n",
        "#normalization: rescale the images from [0,255] to the [0.0,1.0] range.\n",
        "x_train, x_test = x_train/255.0, x_test/255.0\n",
        "print(\"Dimansion of x_train: \", x_train.shape)\n",
        "\n",
        "#reshape and normalize the augmented training set\n",
        "x_train_aug = x_train_aug.reshape(x_train_aug.shape[0], x_train_aug.shape[1]*x_train_aug.shape[2])\n",
        "x_train_aug = x_train_aug/255.0\n",
        "print(\"Dimansion of x_train_aug: \", x_train_aug.shape)\n",
        "\n",
        "#take the first 100 as validation set\n",
        "x_validation = x_train[:100]\n",
        "y_validation = y_train[:100]\n",
        "x_train = x_train[101:]\n",
        "y_train = y_train[101:]\n",
        "\n",
        "#unormalized data\n",
        "#x_train_unormalized = x_train * 255.0\n",
        "#x_test_unormalized = x_test * 255.0\n",
        "#x_validation_unormalized = x_validation * 255.0\n",
        "\n",
        "#At this point x_train, x_test, x_train_unormalized, x_validation_unormalized, x_test_unormalized, x_validation are 2-d np array, y_train, y_test, y_validation are 1-d np array"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Dimansion of x_train:  (60000, 784)\n",
            "Dimansion of x_train_aug:  (120032, 784)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Kj9CrRTlrged"
      },
      "source": [
        "# preprocess data, normalization by substracting training mean from training, validation, test datasets\n",
        "def prepro(X_train, X_val, X_test):\n",
        "    mean = np.mean(X_train)\n",
        "    return X_train - mean, X_val - mean, X_test - mean"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LIV9nDb1CNJo"
      },
      "source": [
        "Hyper-param tuning (Not used)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Xexn-q0OCMWh"
      },
      "source": [
        "def tune_hyperparam(model, n_iter, alpha, mb_size, n_experiment, reg, print_after, p_dropout, loss, nonlin, solver_fun, max_norm, x_train, x_val, x_test, y_train, y_val, y_test, candidates, nlayer):\n",
        "    M, D, C = x_train.shape[0], x_train.shape[1], y_train.max() + 1\n",
        "\n",
        "    accuracies = np.array([[0,0,0,0]]) # initiate accuracy array, containing hyper-param candidates composition and corresponding accuracy\n",
        "    accs = np.zeros(n_experiment) # initiate accuracy array for a single set of hyper-param\n",
        "\n",
        "    for max_norm in candidates[0]:\n",
        "      for dropout_rate_input in candidates[1]:\n",
        "        for dropout_rate_hidden in candidates[2]:\n",
        "          # composite the droup out array based on the number of hidden layers\n",
        "          dropoutArray = np.array([dropout_rate_input])\n",
        "          for i in range(1):\n",
        "            dropoutArray = np.append(dropoutArray, [dropout_rate_hidden])\n",
        "\n",
        "          print('Experiment on hyper-parm candidates. max_norm: ', max_norm, 'input drop out rate: ', dropout_rate_input, 'hidden layer drop out rate: ', dropout_rate_hidden)\n",
        "\n",
        "          for k in range(n_experiment):\n",
        "             print('Experiment-{}'.format(k + 1))\n",
        "\n",
        "             # Reset model\n",
        "             if model == 'ff':\n",
        "                net = nn.FeedForwardNet(D, C, H=2048, lam=reg, p_dropout=dropoutArray, loss=loss, nonlin=nonlin, nlayer=nlayer)\n",
        "             elif model == 'cnn':\n",
        "                net = nn.ConvNet(10, C, H=128, lam=1e-3, p_dropout=dropoutArray, loss=loss, nonlin=nonlin)\n",
        "\n",
        "             net = solver_fun(\n",
        "                 net, x_train, y_train, val_set=(x_val, y_val), mb_size=mb_size, alpha=alpha,\n",
        "                 n_iter=n_iter, print_after=print_after, max_norm=max_norm\n",
        "             )\n",
        "\n",
        "             y_pred = net.predict(x_test)\n",
        "             accs[k] = np.mean(y_pred == y_test)\n",
        "\n",
        "          print()\n",
        "          print('Mean accuracy: {:.4f}, std: {:.4f}'.format(accs.mean(), accs.std()))\n",
        "          print()\n",
        "          \n",
        "          accuracies = np.concatenate((accuracies, [[max_norm, dropout_rate_input, dropout_rate_hidden, accs.mean()]]))\n",
        "\n",
        "      \n",
        "    \n",
        "    index = np.argmax(accuracies, axis=0)[3] # index of the best hyper-parm in the accuracies\n",
        "    best_max_norm = accuracies[index][0]\n",
        "    best_dropout_rate_input = accuracies[index][1]\n",
        "    best_dropout_rate_hidden = accuracies[index][2]\n",
        "    best_acc = accuracies[index][3]\n",
        "\n",
        "         \n",
        "    return best_max_norm, best_dropout_rate_input, best_dropout_rate_hidden, best_acc\n",
        "\n"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HudecacFZKBo"
      },
      "source": [
        "# Train and predict mnist dataset\n",
        "def run_mnist(model, n_iter, alpha, mb_size, n_experiment, reg, print_after, p_dropout, loss, nonlin, solver, max_norm, x_train, x_val, x_test, y_train, y_val, y_test, candidates, nlayer):\n",
        "    M, D, C = x_train.shape[0], x_train.shape[1], y_train.max() + 1\n",
        "  \n",
        "    x_train, x_val, x_test = prepro(x_train, x_val, x_test)\n",
        "    \n",
        "    # A key/value hash table of optimization algorithm, all algorithms use minibatch technique\n",
        "    solvers = dict(\n",
        "        sgd=sgd,                # SGD\n",
        "        momentum=momentum,          # Momentum SGD\n",
        "        nesterov=nesterov,          # Nesterov Momentum\n",
        "        adagrad=adagrad,           # Adagrad\n",
        "        rmsprop=rmsprop,          # RMSprop\n",
        "        adam=adam             # Adam\n",
        "    )\n",
        "    \n",
        "    solver_fun = solvers[solver]\n",
        "    \n",
        "    print()\n",
        "    print('Experimenting on {}'.format(solver))\n",
        "    print()\n",
        "    \n",
        "    if model == 'cnn':\n",
        "        img_shape = (1, 28, 28)\n",
        "        x_train = x_train.reshape(-1, *img_shape)\n",
        "        x_val = x_val.reshape(-1, *img_shape)\n",
        "        x_test = x_test.reshape(-1, *img_shape)\n",
        "\n",
        "    #tune hyper-parameters\n",
        "    best_max_norm, best_dropout_rate_input, best_dropout_rate_hidden, best_acc = tune_hyperparam(model, 500, alpha, mb_size, n_experiment, reg, print_after, p_dropout, loss, nonlin, solver_fun, max_norm, x_train, x_val, x_test, y_train, y_val, y_test, candidates, nlayer)\n",
        "\n",
        "    #train and predict\n",
        "    accs = np.zeros(n_experiment) # initiate accuracy array for a single set of hyper-param\n",
        "    best_dropoutArray = np.array([best_dropout_rate_input])\n",
        "    for i in range(1):\n",
        "        best_dropoutArray = np.append(best_dropoutArray, [best_dropout_rate_hidden])\n",
        "\n",
        "    for k in range(n_experiment):\n",
        "        print('Experiment-{}'.format(k + 1))\n",
        "\n",
        "        # Reset model\n",
        "        if model == 'ff':\n",
        "           net = nn.FeedForwardNet(D, C, H=2048, lam=reg, p_dropout=best_dropoutArray, loss=loss, nonlin=nonlin, nlayer=nlayer)\n",
        "        elif model == 'cnn':\n",
        "           net = nn.ConvNet(10, C, H=1024, lam=1e-3, p_dropout=best_dropoutArray, loss=loss, nonlin=nonlin)\n",
        "\n",
        "        net = solver_fun(\n",
        "            net, x_train, y_train, val_set=(x_val, y_val), mb_size=mb_size, alpha=alpha,\n",
        "            n_iter=n_iter, print_after=print_after, max_norm=best_max_norm\n",
        "        )\n",
        "\n",
        "        y_pred = net.predict(x_test)\n",
        "        accs[k] = np.mean(y_pred == y_test)\n",
        "\n",
        "    print()\n",
        "    print('Best max_norm: ', best_max_norm, 'best dropout rate for input: ', best_dropout_rate_input, 'best dropout rate for hidden layers: ', best_dropout_rate_hidden, 'and result accuracy is: ', accs.mean())\n",
        "\n"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vQqm4nrZuZXn"
      },
      "source": [
        "Test"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9J-0YwZldjij"
      },
      "source": [
        "Dropout 3-layer CNN with 1024 hidden units\n",
        "(Conv layer with stride=1, padding=1; Maxpool; Fully connected with relu) \n",
        "+ max-norm constraint "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1vHj96cHdoSA",
        "outputId": "088a7feb-7c82-49c3-9289-96fbad2369be"
      },
      "source": [
        "model = 'cnn'\n",
        "n_iter = 50000\n",
        "alpha = 1e-3 #learning rate\n",
        "mb_size = 64\n",
        "n_experiment = 1\n",
        "reg = 1e-5\n",
        "print_after = 100\n",
        "p_dropout = [0.5,0.8] #only 0.5 is used since only dropout is applied for the only one fully connected layer\n",
        "loss = 'cross_ent'\n",
        "nonlin = 'relu'\n",
        "solver = 'sgd'\n",
        "max_norm = 4\n",
        "nlayer = 3\n",
        "\n",
        "# Define default candidates\n",
        "candidates = np.array([[3, 3.5, 4],[0.5, 0.65, 0.7, 0.8],[0.8]]) # first hyper-param as max-norm, second as drop-out rate for input layer, third as drop-out rate for hidden layers\n",
        "\n",
        "run_mnist(model, n_iter, alpha, mb_size, n_experiment, reg, print_after, p_dropout, loss, nonlin, solver, max_norm, x_train, x_validation, x_test, y_train, y_validation, y_test, candidates, nlayer)"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:16: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray\n",
            "  app.launch_new_instance()\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Experimenting on sgd\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  3 input drop out rate:  0.5 hidden layer drop out rate:  0.8\n",
            "Experiment-1\n",
            "Iter-100 loss: 2.2404 validation: 0.460000\n",
            "Iter-200 loss: 2.1985 validation: 0.610000\n",
            "Iter-300 loss: 2.1318 validation: 0.650000\n",
            "Iter-400 loss: 2.1099 validation: 0.730000\n",
            "Iter-500 loss: 1.9789 validation: 0.730000\n",
            "\n",
            "Mean accuracy: 0.7056, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  3 input drop out rate:  0.65 hidden layer drop out rate:  0.8\n",
            "Experiment-1\n",
            "Iter-100 loss: 2.2592 validation: 0.500000\n",
            "Iter-200 loss: 2.1815 validation: 0.640000\n",
            "Iter-300 loss: 2.1350 validation: 0.730000\n",
            "Iter-400 loss: 2.0410 validation: 0.800000\n",
            "Iter-500 loss: 1.9353 validation: 0.810000\n",
            "\n",
            "Mean accuracy: 0.7353, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  3 input drop out rate:  0.7 hidden layer drop out rate:  0.8\n",
            "Experiment-1\n",
            "Iter-100 loss: 2.2778 validation: 0.410000\n",
            "Iter-200 loss: 2.2431 validation: 0.640000\n",
            "Iter-300 loss: 2.2226 validation: 0.740000\n",
            "Iter-400 loss: 2.1581 validation: 0.770000\n",
            "Iter-500 loss: 2.1075 validation: 0.750000\n",
            "\n",
            "Mean accuracy: 0.7101, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  3 input drop out rate:  0.8 hidden layer drop out rate:  0.8\n",
            "Experiment-1\n",
            "Iter-100 loss: 2.2711 validation: 0.290000\n",
            "Iter-200 loss: 2.2311 validation: 0.590000\n",
            "Iter-300 loss: 2.1771 validation: 0.730000\n",
            "Iter-400 loss: 2.0606 validation: 0.820000\n",
            "Iter-500 loss: 2.0025 validation: 0.810000\n",
            "\n",
            "Mean accuracy: 0.7485, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  3.5 input drop out rate:  0.5 hidden layer drop out rate:  0.8\n",
            "Experiment-1\n",
            "Iter-100 loss: 2.1489 validation: 0.610000\n",
            "Iter-200 loss: 1.9590 validation: 0.730000\n",
            "Iter-300 loss: 1.7967 validation: 0.840000\n",
            "Iter-400 loss: 1.6433 validation: 0.850000\n",
            "Iter-500 loss: 1.5345 validation: 0.840000\n",
            "\n",
            "Mean accuracy: 0.7832, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  3.5 input drop out rate:  0.65 hidden layer drop out rate:  0.8\n",
            "Experiment-1\n",
            "Iter-100 loss: 2.1641 validation: 0.610000\n",
            "Iter-200 loss: 1.9693 validation: 0.810000\n",
            "Iter-300 loss: 1.8106 validation: 0.800000\n",
            "Iter-400 loss: 1.6843 validation: 0.820000\n",
            "Iter-500 loss: 1.5641 validation: 0.830000\n",
            "\n",
            "Mean accuracy: 0.7892, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  3.5 input drop out rate:  0.7 hidden layer drop out rate:  0.8\n",
            "Experiment-1\n",
            "Iter-100 loss: 2.2378 validation: 0.560000\n",
            "Iter-200 loss: 2.1232 validation: 0.740000\n",
            "Iter-300 loss: 2.0239 validation: 0.750000\n",
            "Iter-400 loss: 1.9492 validation: 0.760000\n",
            "Iter-500 loss: 1.8072 validation: 0.750000\n",
            "\n",
            "Mean accuracy: 0.7384, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  3.5 input drop out rate:  0.8 hidden layer drop out rate:  0.8\n",
            "Experiment-1\n",
            "Iter-100 loss: 2.0734 validation: 0.600000\n",
            "Iter-200 loss: 1.8467 validation: 0.710000\n",
            "Iter-300 loss: 1.7841 validation: 0.760000\n",
            "Iter-400 loss: 1.4458 validation: 0.820000\n",
            "Iter-500 loss: 1.3960 validation: 0.820000\n",
            "\n",
            "Mean accuracy: 0.7915, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  4 input drop out rate:  0.5 hidden layer drop out rate:  0.8\n",
            "Experiment-1\n",
            "Iter-100 loss: 2.1657 validation: 0.590000\n",
            "Iter-200 loss: 2.0621 validation: 0.760000\n",
            "Iter-300 loss: 1.9500 validation: 0.780000\n",
            "Iter-400 loss: 1.7803 validation: 0.790000\n",
            "Iter-500 loss: 1.6891 validation: 0.830000\n",
            "\n",
            "Mean accuracy: 0.8013, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  4 input drop out rate:  0.65 hidden layer drop out rate:  0.8\n",
            "Experiment-1\n",
            "Iter-100 loss: 2.0984 validation: 0.620000\n",
            "Iter-200 loss: 1.9761 validation: 0.730000\n",
            "Iter-300 loss: 1.7195 validation: 0.780000\n",
            "Iter-400 loss: 1.5310 validation: 0.830000\n",
            "Iter-500 loss: 1.4930 validation: 0.840000\n",
            "\n",
            "Mean accuracy: 0.8010, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  4 input drop out rate:  0.7 hidden layer drop out rate:  0.8\n",
            "Experiment-1\n",
            "Iter-100 loss: 2.0704 validation: 0.740000\n",
            "Iter-200 loss: 1.8197 validation: 0.860000\n",
            "Iter-300 loss: 1.5496 validation: 0.840000\n",
            "Iter-400 loss: 1.4880 validation: 0.850000\n",
            "Iter-500 loss: 1.3003 validation: 0.840000\n",
            "\n",
            "Mean accuracy: 0.8079, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  4 input drop out rate:  0.8 hidden layer drop out rate:  0.8\n",
            "Experiment-1\n",
            "Iter-100 loss: 2.1614 validation: 0.580000\n",
            "Iter-200 loss: 1.9460 validation: 0.780000\n",
            "Iter-300 loss: 1.7967 validation: 0.800000\n",
            "Iter-400 loss: 1.5875 validation: 0.810000\n",
            "Iter-500 loss: 1.3903 validation: 0.820000\n",
            "\n",
            "Mean accuracy: 0.7884, std: 0.0000\n",
            "\n",
            "Experiment-1\n",
            "Iter-100 loss: 2.1616 validation: 0.720000\n",
            "Iter-200 loss: 2.0265 validation: 0.780000\n",
            "Iter-300 loss: 1.8532 validation: 0.790000\n",
            "Iter-400 loss: 1.7659 validation: 0.830000\n",
            "Iter-500 loss: 1.5387 validation: 0.810000\n",
            "Iter-600 loss: 1.5122 validation: 0.830000\n",
            "Iter-700 loss: 1.2564 validation: 0.840000\n",
            "Iter-800 loss: 1.3059 validation: 0.840000\n",
            "Iter-900 loss: 1.2292 validation: 0.840000\n",
            "Iter-1000 loss: 1.0214 validation: 0.850000\n",
            "Iter-1100 loss: 0.9713 validation: 0.860000\n",
            "Iter-1200 loss: 0.9289 validation: 0.850000\n",
            "Iter-1300 loss: 0.7347 validation: 0.870000\n",
            "Iter-1400 loss: 0.6925 validation: 0.880000\n",
            "Iter-1500 loss: 0.7871 validation: 0.880000\n",
            "Iter-1600 loss: 0.7500 validation: 0.880000\n",
            "Iter-1700 loss: 0.8242 validation: 0.890000\n",
            "Iter-1800 loss: 0.7684 validation: 0.890000\n",
            "Iter-1900 loss: 0.6402 validation: 0.890000\n",
            "Iter-2000 loss: 0.6324 validation: 0.900000\n",
            "Iter-2100 loss: 0.5094 validation: 0.900000\n",
            "Iter-2200 loss: 0.6392 validation: 0.900000\n",
            "Iter-2300 loss: 0.6600 validation: 0.900000\n",
            "Iter-2400 loss: 0.5922 validation: 0.900000\n",
            "Iter-2500 loss: 0.6005 validation: 0.910000\n",
            "Iter-2600 loss: 0.5450 validation: 0.900000\n",
            "Iter-2700 loss: 0.4336 validation: 0.910000\n",
            "Iter-2800 loss: 0.5898 validation: 0.900000\n",
            "Iter-2900 loss: 0.6195 validation: 0.910000\n",
            "Iter-3000 loss: 0.5486 validation: 0.910000\n",
            "Iter-3100 loss: 0.4183 validation: 0.910000\n",
            "Iter-3200 loss: 0.4117 validation: 0.900000\n",
            "Iter-3300 loss: 0.4679 validation: 0.900000\n",
            "Iter-3400 loss: 0.4353 validation: 0.910000\n",
            "Iter-3500 loss: 0.4687 validation: 0.900000\n",
            "Iter-3600 loss: 0.4814 validation: 0.910000\n",
            "Iter-3700 loss: 0.3742 validation: 0.910000\n",
            "Iter-3800 loss: 0.4101 validation: 0.900000\n",
            "Iter-3900 loss: 0.6302 validation: 0.910000\n",
            "Iter-4000 loss: 0.3599 validation: 0.910000\n",
            "Iter-4100 loss: 0.5201 validation: 0.910000\n",
            "Iter-4200 loss: 0.4826 validation: 0.910000\n",
            "Iter-4300 loss: 0.3376 validation: 0.910000\n",
            "Iter-4400 loss: 0.4957 validation: 0.910000\n",
            "Iter-4500 loss: 0.4668 validation: 0.920000\n",
            "Iter-4600 loss: 0.6193 validation: 0.920000\n",
            "Iter-4700 loss: 0.4493 validation: 0.930000\n",
            "Iter-4800 loss: 0.3816 validation: 0.930000\n",
            "Iter-4900 loss: 0.4774 validation: 0.930000\n",
            "Iter-5000 loss: 0.3777 validation: 0.930000\n",
            "Iter-5100 loss: 0.3538 validation: 0.930000\n",
            "Iter-5200 loss: 0.3412 validation: 0.930000\n",
            "Iter-5300 loss: 0.4194 validation: 0.930000\n",
            "Iter-5400 loss: 0.3710 validation: 0.930000\n",
            "Iter-5500 loss: 0.3552 validation: 0.930000\n",
            "Iter-5600 loss: 0.4231 validation: 0.930000\n",
            "Iter-5700 loss: 0.4507 validation: 0.930000\n",
            "Iter-5800 loss: 0.5553 validation: 0.930000\n",
            "Iter-5900 loss: 0.3452 validation: 0.930000\n",
            "Iter-6000 loss: 0.2898 validation: 0.930000\n",
            "Iter-6100 loss: 0.4919 validation: 0.930000\n",
            "Iter-6200 loss: 0.2394 validation: 0.930000\n",
            "Iter-6300 loss: 0.3086 validation: 0.930000\n",
            "Iter-6400 loss: 0.3606 validation: 0.930000\n",
            "Iter-6500 loss: 0.3350 validation: 0.930000\n",
            "Iter-6600 loss: 0.4557 validation: 0.930000\n",
            "Iter-6700 loss: 0.4419 validation: 0.930000\n",
            "Iter-6800 loss: 0.2958 validation: 0.930000\n",
            "Iter-6900 loss: 0.4555 validation: 0.930000\n",
            "Iter-7000 loss: 0.4132 validation: 0.930000\n",
            "Iter-7100 loss: 0.2516 validation: 0.930000\n",
            "Iter-7200 loss: 0.3822 validation: 0.930000\n",
            "Iter-7300 loss: 0.2840 validation: 0.930000\n",
            "Iter-7400 loss: 0.4366 validation: 0.930000\n",
            "Iter-7500 loss: 0.3388 validation: 0.930000\n",
            "Iter-7600 loss: 0.2928 validation: 0.930000\n",
            "Iter-7700 loss: 0.3386 validation: 0.930000\n",
            "Iter-7800 loss: 0.5202 validation: 0.930000\n",
            "Iter-7900 loss: 0.2462 validation: 0.930000\n",
            "Iter-8000 loss: 0.3233 validation: 0.930000\n",
            "Iter-8100 loss: 0.3031 validation: 0.930000\n",
            "Iter-8200 loss: 0.4606 validation: 0.930000\n",
            "Iter-8300 loss: 0.4582 validation: 0.940000\n",
            "Iter-8400 loss: 0.4128 validation: 0.930000\n",
            "Iter-8500 loss: 0.3553 validation: 0.930000\n",
            "Iter-8600 loss: 0.3965 validation: 0.930000\n",
            "Iter-8700 loss: 0.3214 validation: 0.930000\n",
            "Iter-8800 loss: 0.3509 validation: 0.930000\n",
            "Iter-8900 loss: 0.4089 validation: 0.930000\n",
            "Iter-9000 loss: 0.2085 validation: 0.930000\n",
            "Iter-9100 loss: 0.4747 validation: 0.930000\n",
            "Iter-9200 loss: 0.4298 validation: 0.930000\n",
            "Iter-9300 loss: 0.2603 validation: 0.930000\n",
            "Iter-9400 loss: 0.2878 validation: 0.930000\n",
            "Iter-9500 loss: 0.4025 validation: 0.930000\n",
            "Iter-9600 loss: 0.3394 validation: 0.930000\n",
            "Iter-9700 loss: 0.2543 validation: 0.930000\n",
            "Iter-9800 loss: 0.3304 validation: 0.930000\n",
            "Iter-9900 loss: 0.3111 validation: 0.930000\n",
            "Iter-10000 loss: 0.2341 validation: 0.930000\n",
            "Iter-10100 loss: 0.3658 validation: 0.930000\n",
            "Iter-10200 loss: 0.3164 validation: 0.930000\n",
            "Iter-10300 loss: 0.2575 validation: 0.930000\n",
            "Iter-10400 loss: 0.2684 validation: 0.930000\n",
            "Iter-10500 loss: 0.2790 validation: 0.930000\n",
            "Iter-10600 loss: 0.3354 validation: 0.940000\n",
            "Iter-10700 loss: 0.3296 validation: 0.930000\n",
            "Iter-10800 loss: 0.1910 validation: 0.940000\n",
            "Iter-10900 loss: 0.4178 validation: 0.930000\n",
            "Iter-11000 loss: 0.3857 validation: 0.930000\n",
            "Iter-11100 loss: 0.3087 validation: 0.940000\n",
            "Iter-11200 loss: 0.3025 validation: 0.930000\n",
            "Iter-11300 loss: 0.3258 validation: 0.930000\n",
            "Iter-11400 loss: 0.3035 validation: 0.930000\n",
            "Iter-11500 loss: 0.3897 validation: 0.930000\n",
            "Iter-11600 loss: 0.2197 validation: 0.930000\n",
            "Iter-11700 loss: 0.3829 validation: 0.930000\n",
            "Iter-11800 loss: 0.3643 validation: 0.930000\n",
            "Iter-11900 loss: 0.1618 validation: 0.930000\n",
            "Iter-12000 loss: 0.2574 validation: 0.940000\n",
            "Iter-12100 loss: 0.3203 validation: 0.940000\n",
            "Iter-12200 loss: 0.1631 validation: 0.930000\n",
            "Iter-12300 loss: 0.2075 validation: 0.930000\n",
            "Iter-12400 loss: 0.2671 validation: 0.930000\n",
            "Iter-12500 loss: 0.2693 validation: 0.930000\n",
            "Iter-12600 loss: 0.2498 validation: 0.930000\n",
            "Iter-12700 loss: 0.3815 validation: 0.930000\n",
            "Iter-12800 loss: 0.3423 validation: 0.930000\n",
            "Iter-12900 loss: 0.2579 validation: 0.940000\n",
            "Iter-13000 loss: 0.2403 validation: 0.940000\n",
            "Iter-13100 loss: 0.2712 validation: 0.930000\n",
            "Iter-13200 loss: 0.3925 validation: 0.930000\n",
            "Iter-13300 loss: 0.2964 validation: 0.940000\n",
            "Iter-13400 loss: 0.4039 validation: 0.940000\n",
            "Iter-13500 loss: 0.2586 validation: 0.930000\n",
            "Iter-13600 loss: 0.2825 validation: 0.930000\n",
            "Iter-13700 loss: 0.2214 validation: 0.940000\n",
            "Iter-13800 loss: 0.3382 validation: 0.940000\n",
            "Iter-13900 loss: 0.2088 validation: 0.940000\n",
            "Iter-14000 loss: 0.3026 validation: 0.940000\n",
            "Iter-14100 loss: 0.3132 validation: 0.930000\n",
            "Iter-14200 loss: 0.2804 validation: 0.940000\n",
            "Iter-14300 loss: 0.3325 validation: 0.940000\n",
            "Iter-14400 loss: 0.4722 validation: 0.940000\n",
            "Iter-14500 loss: 0.2701 validation: 0.940000\n",
            "Iter-14600 loss: 0.4282 validation: 0.940000\n",
            "Iter-14700 loss: 0.2686 validation: 0.930000\n",
            "Iter-14800 loss: 0.1987 validation: 0.940000\n",
            "Iter-14900 loss: 0.1805 validation: 0.940000\n",
            "Iter-15000 loss: 0.4078 validation: 0.940000\n",
            "Iter-15100 loss: 0.2830 validation: 0.940000\n",
            "Iter-15200 loss: 0.3749 validation: 0.940000\n",
            "Iter-15300 loss: 0.3078 validation: 0.940000\n",
            "Iter-15400 loss: 0.4124 validation: 0.940000\n",
            "Iter-15500 loss: 0.1763 validation: 0.940000\n",
            "Iter-15600 loss: 0.2677 validation: 0.940000\n",
            "Iter-15700 loss: 0.1462 validation: 0.940000\n",
            "Iter-15800 loss: 0.2507 validation: 0.940000\n",
            "Iter-15900 loss: 0.1771 validation: 0.940000\n",
            "Iter-16000 loss: 0.2175 validation: 0.940000\n",
            "Iter-16100 loss: 0.3762 validation: 0.940000\n",
            "Iter-16200 loss: 0.3063 validation: 0.940000\n",
            "Iter-16300 loss: 0.3527 validation: 0.940000\n",
            "Iter-16400 loss: 0.4436 validation: 0.940000\n",
            "Iter-16500 loss: 0.5094 validation: 0.940000\n",
            "Iter-16600 loss: 0.2828 validation: 0.940000\n",
            "Iter-16700 loss: 0.2531 validation: 0.940000\n",
            "Iter-16800 loss: 0.5449 validation: 0.940000\n",
            "Iter-16900 loss: 0.2977 validation: 0.940000\n",
            "Iter-17000 loss: 0.2484 validation: 0.940000\n",
            "Iter-17100 loss: 0.3045 validation: 0.940000\n",
            "Iter-17200 loss: 0.2493 validation: 0.940000\n",
            "Iter-17300 loss: 0.1848 validation: 0.940000\n",
            "Iter-17400 loss: 0.2107 validation: 0.940000\n",
            "Iter-17500 loss: 0.2488 validation: 0.940000\n",
            "Iter-17600 loss: 0.3747 validation: 0.940000\n",
            "Iter-17700 loss: 0.2199 validation: 0.940000\n",
            "Iter-17800 loss: 0.2032 validation: 0.940000\n",
            "Iter-17900 loss: 0.2413 validation: 0.940000\n",
            "Iter-18000 loss: 0.3190 validation: 0.940000\n",
            "Iter-18100 loss: 0.2893 validation: 0.940000\n",
            "Iter-18200 loss: 0.2797 validation: 0.940000\n",
            "Iter-18300 loss: 0.3466 validation: 0.940000\n",
            "Iter-18400 loss: 0.1267 validation: 0.940000\n",
            "Iter-18500 loss: 0.2414 validation: 0.940000\n",
            "Iter-18600 loss: 0.3530 validation: 0.940000\n",
            "Iter-18700 loss: 0.2091 validation: 0.940000\n",
            "Iter-18800 loss: 0.3107 validation: 0.940000\n",
            "Iter-18900 loss: 0.2321 validation: 0.940000\n",
            "Iter-19000 loss: 0.3557 validation: 0.940000\n",
            "Iter-19100 loss: 0.1591 validation: 0.940000\n",
            "Iter-19200 loss: 0.2565 validation: 0.940000\n",
            "Iter-19300 loss: 0.2308 validation: 0.940000\n",
            "Iter-19400 loss: 0.3135 validation: 0.940000\n",
            "Iter-19500 loss: 0.1867 validation: 0.950000\n",
            "Iter-19600 loss: 0.2174 validation: 0.940000\n",
            "Iter-19700 loss: 0.3448 validation: 0.940000\n",
            "Iter-19800 loss: 0.3335 validation: 0.950000\n",
            "Iter-19900 loss: 0.2174 validation: 0.940000\n",
            "Iter-20000 loss: 0.2463 validation: 0.950000\n",
            "Iter-20100 loss: 0.2317 validation: 0.940000\n",
            "Iter-20200 loss: 0.1984 validation: 0.950000\n",
            "Iter-20300 loss: 0.2067 validation: 0.940000\n",
            "Iter-20400 loss: 0.2671 validation: 0.950000\n",
            "Iter-20500 loss: 0.1091 validation: 0.950000\n",
            "Iter-20600 loss: 0.2734 validation: 0.950000\n",
            "Iter-20700 loss: 0.2707 validation: 0.950000\n",
            "Iter-20800 loss: 0.1879 validation: 0.950000\n",
            "Iter-20900 loss: 0.2913 validation: 0.950000\n",
            "Iter-21000 loss: 0.2116 validation: 0.950000\n",
            "Iter-21100 loss: 0.3923 validation: 0.950000\n",
            "Iter-21200 loss: 0.2093 validation: 0.950000\n",
            "Iter-21300 loss: 0.2917 validation: 0.950000\n",
            "Iter-21400 loss: 0.3946 validation: 0.950000\n",
            "Iter-21500 loss: 0.1675 validation: 0.950000\n",
            "Iter-21600 loss: 0.2726 validation: 0.950000\n",
            "Iter-21700 loss: 0.1646 validation: 0.950000\n",
            "Iter-21800 loss: 0.4076 validation: 0.950000\n",
            "Iter-21900 loss: 0.3619 validation: 0.950000\n",
            "Iter-22000 loss: 0.3195 validation: 0.950000\n",
            "Iter-22100 loss: 0.2649 validation: 0.950000\n",
            "Iter-22200 loss: 0.2742 validation: 0.950000\n",
            "Iter-22300 loss: 0.2745 validation: 0.950000\n",
            "Iter-22400 loss: 0.3312 validation: 0.950000\n",
            "Iter-22500 loss: 0.1826 validation: 0.950000\n",
            "Iter-22600 loss: 0.4732 validation: 0.950000\n",
            "Iter-22700 loss: 0.3064 validation: 0.950000\n",
            "Iter-22800 loss: 0.1846 validation: 0.950000\n",
            "Iter-22900 loss: 0.2397 validation: 0.950000\n",
            "Iter-23000 loss: 0.2711 validation: 0.950000\n",
            "Iter-23100 loss: 0.1902 validation: 0.950000\n",
            "Iter-23200 loss: 0.2649 validation: 0.950000\n",
            "Iter-23300 loss: 0.2492 validation: 0.950000\n",
            "Iter-23400 loss: 0.2641 validation: 0.950000\n",
            "Iter-23500 loss: 0.2480 validation: 0.950000\n",
            "Iter-23600 loss: 0.3797 validation: 0.950000\n",
            "Iter-23700 loss: 0.1311 validation: 0.950000\n",
            "Iter-23800 loss: 0.3710 validation: 0.950000\n",
            "Iter-23900 loss: 0.3009 validation: 0.950000\n",
            "Iter-24000 loss: 0.2679 validation: 0.950000\n",
            "Iter-24100 loss: 0.3119 validation: 0.950000\n",
            "Iter-24200 loss: 0.2853 validation: 0.950000\n",
            "Iter-24300 loss: 0.2210 validation: 0.950000\n",
            "Iter-24400 loss: 0.1821 validation: 0.950000\n",
            "Iter-24500 loss: 0.2985 validation: 0.950000\n",
            "Iter-24600 loss: 0.2364 validation: 0.950000\n",
            "Iter-24700 loss: 0.2191 validation: 0.950000\n",
            "Iter-24800 loss: 0.1252 validation: 0.950000\n",
            "Iter-24900 loss: 0.2246 validation: 0.950000\n",
            "Iter-25000 loss: 0.3018 validation: 0.950000\n",
            "Iter-25100 loss: 0.2160 validation: 0.950000\n",
            "Iter-25200 loss: 0.1975 validation: 0.950000\n",
            "Iter-25300 loss: 0.1682 validation: 0.950000\n",
            "Iter-25400 loss: 0.2145 validation: 0.950000\n",
            "Iter-25500 loss: 0.1797 validation: 0.950000\n",
            "Iter-25600 loss: 0.1765 validation: 0.950000\n",
            "Iter-25700 loss: 0.3077 validation: 0.950000\n",
            "Iter-25800 loss: 0.1711 validation: 0.950000\n",
            "Iter-25900 loss: 0.2773 validation: 0.950000\n",
            "Iter-26000 loss: 0.3502 validation: 0.950000\n",
            "Iter-26100 loss: 0.2686 validation: 0.950000\n",
            "Iter-26200 loss: 0.1767 validation: 0.950000\n",
            "Iter-26300 loss: 0.1731 validation: 0.950000\n",
            "Iter-26400 loss: 0.2682 validation: 0.950000\n",
            "Iter-26500 loss: 0.3933 validation: 0.950000\n",
            "Iter-26600 loss: 0.3088 validation: 0.950000\n",
            "Iter-26700 loss: 0.1151 validation: 0.950000\n",
            "Iter-26800 loss: 0.4177 validation: 0.950000\n",
            "Iter-26900 loss: 0.3106 validation: 0.950000\n",
            "Iter-27000 loss: 0.3928 validation: 0.950000\n",
            "Iter-27100 loss: 0.2665 validation: 0.950000\n",
            "Iter-27200 loss: 0.2367 validation: 0.950000\n",
            "Iter-27300 loss: 0.2584 validation: 0.950000\n",
            "Iter-27400 loss: 0.1226 validation: 0.950000\n",
            "Iter-27500 loss: 0.2830 validation: 0.950000\n",
            "Iter-27600 loss: 0.2707 validation: 0.950000\n",
            "Iter-27700 loss: 0.3474 validation: 0.950000\n",
            "Iter-27800 loss: 0.2717 validation: 0.950000\n",
            "Iter-27900 loss: 0.2889 validation: 0.950000\n",
            "Iter-28000 loss: 0.2188 validation: 0.950000\n",
            "Iter-28100 loss: 0.2540 validation: 0.950000\n",
            "Iter-28200 loss: 0.2905 validation: 0.950000\n",
            "Iter-28300 loss: 0.2522 validation: 0.950000\n",
            "Iter-28400 loss: 0.2368 validation: 0.950000\n",
            "Iter-28500 loss: 0.2807 validation: 0.950000\n",
            "Iter-28600 loss: 0.1742 validation: 0.950000\n",
            "Iter-28700 loss: 0.1701 validation: 0.950000\n",
            "Iter-28800 loss: 0.2162 validation: 0.960000\n",
            "Iter-28900 loss: 0.3434 validation: 0.950000\n",
            "Iter-29000 loss: 0.3612 validation: 0.950000\n",
            "Iter-29100 loss: 0.1249 validation: 0.950000\n",
            "Iter-29200 loss: 0.3187 validation: 0.950000\n",
            "Iter-29300 loss: 0.3965 validation: 0.950000\n",
            "Iter-29400 loss: 0.1137 validation: 0.950000\n",
            "Iter-29500 loss: 0.2425 validation: 0.950000\n",
            "Iter-29600 loss: 0.2076 validation: 0.950000\n",
            "Iter-29700 loss: 0.2195 validation: 0.950000\n",
            "Iter-29800 loss: 0.2251 validation: 0.950000\n",
            "Iter-29900 loss: 0.4114 validation: 0.950000\n",
            "Iter-30000 loss: 0.1664 validation: 0.950000\n",
            "Iter-30100 loss: 0.1910 validation: 0.950000\n",
            "Iter-30200 loss: 0.1816 validation: 0.950000\n",
            "Iter-30300 loss: 0.3273 validation: 0.950000\n",
            "Iter-30400 loss: 0.1676 validation: 0.950000\n",
            "Iter-30500 loss: 0.3415 validation: 0.950000\n",
            "Iter-30600 loss: 0.2403 validation: 0.950000\n",
            "Iter-30700 loss: 0.2017 validation: 0.960000\n",
            "Iter-30800 loss: 0.1469 validation: 0.950000\n",
            "Iter-30900 loss: 0.3174 validation: 0.950000\n",
            "Iter-31000 loss: 0.1306 validation: 0.950000\n",
            "Iter-31100 loss: 0.3487 validation: 0.950000\n",
            "Iter-31200 loss: 0.4161 validation: 0.950000\n",
            "Iter-31300 loss: 0.2320 validation: 0.950000\n",
            "Iter-31400 loss: 0.1950 validation: 0.950000\n",
            "Iter-31500 loss: 0.1202 validation: 0.950000\n",
            "Iter-31600 loss: 0.3946 validation: 0.950000\n",
            "Iter-31700 loss: 0.3407 validation: 0.950000\n",
            "Iter-31800 loss: 0.1928 validation: 0.960000\n",
            "Iter-31900 loss: 0.1795 validation: 0.950000\n",
            "Iter-32000 loss: 0.2798 validation: 0.950000\n",
            "Iter-32100 loss: 0.1250 validation: 0.950000\n",
            "Iter-32200 loss: 0.0963 validation: 0.950000\n",
            "Iter-32300 loss: 0.2610 validation: 0.950000\n",
            "Iter-32400 loss: 0.2120 validation: 0.950000\n",
            "Iter-32500 loss: 0.2863 validation: 0.950000\n",
            "Iter-32600 loss: 0.1852 validation: 0.950000\n",
            "Iter-32700 loss: 0.1481 validation: 0.960000\n",
            "Iter-32800 loss: 0.1766 validation: 0.950000\n",
            "Iter-32900 loss: 0.1297 validation: 0.950000\n",
            "Iter-33000 loss: 0.2216 validation: 0.950000\n",
            "Iter-33100 loss: 0.2644 validation: 0.950000\n",
            "Iter-33200 loss: 0.1921 validation: 0.950000\n",
            "Iter-33300 loss: 0.1380 validation: 0.950000\n",
            "Iter-33400 loss: 0.2729 validation: 0.950000\n",
            "Iter-33500 loss: 0.2670 validation: 0.950000\n",
            "Iter-33600 loss: 0.2801 validation: 0.950000\n",
            "Iter-33700 loss: 0.1492 validation: 0.950000\n",
            "Iter-33800 loss: 0.2921 validation: 0.950000\n",
            "Iter-33900 loss: 0.1353 validation: 0.950000\n",
            "Iter-34000 loss: 0.2503 validation: 0.950000\n",
            "Iter-34100 loss: 0.0874 validation: 0.960000\n",
            "Iter-34200 loss: 0.1643 validation: 0.960000\n",
            "Iter-34300 loss: 0.1330 validation: 0.960000\n",
            "Iter-34400 loss: 0.1315 validation: 0.950000\n",
            "Iter-34500 loss: 0.2624 validation: 0.970000\n",
            "Iter-34600 loss: 0.3315 validation: 0.950000\n",
            "Iter-34700 loss: 0.2220 validation: 0.960000\n",
            "Iter-34800 loss: 0.1270 validation: 0.950000\n",
            "Iter-34900 loss: 0.2379 validation: 0.950000\n",
            "Iter-35000 loss: 0.1934 validation: 0.960000\n",
            "Iter-35100 loss: 0.2088 validation: 0.960000\n",
            "Iter-35200 loss: 0.2592 validation: 0.950000\n",
            "Iter-35300 loss: 0.3354 validation: 0.960000\n",
            "Iter-35400 loss: 0.2233 validation: 0.960000\n",
            "Iter-35500 loss: 0.2252 validation: 0.960000\n",
            "Iter-35600 loss: 0.2743 validation: 0.950000\n",
            "Iter-35700 loss: 0.1423 validation: 0.960000\n",
            "Iter-35800 loss: 0.2147 validation: 0.960000\n",
            "Iter-35900 loss: 0.3436 validation: 0.960000\n",
            "Iter-36000 loss: 0.1864 validation: 0.950000\n",
            "Iter-36100 loss: 0.2309 validation: 0.960000\n",
            "Iter-36200 loss: 0.2036 validation: 0.950000\n",
            "Iter-36300 loss: 0.5398 validation: 0.960000\n",
            "Iter-36400 loss: 0.2662 validation: 0.960000\n",
            "Iter-36500 loss: 0.2611 validation: 0.960000\n",
            "Iter-36600 loss: 0.1436 validation: 0.960000\n",
            "Iter-36700 loss: 0.1469 validation: 0.950000\n",
            "Iter-36800 loss: 0.3530 validation: 0.960000\n",
            "Iter-36900 loss: 0.3185 validation: 0.960000\n",
            "Iter-37000 loss: 0.1769 validation: 0.960000\n",
            "Iter-37100 loss: 0.3458 validation: 0.960000\n",
            "Iter-37200 loss: 0.1019 validation: 0.970000\n",
            "Iter-37300 loss: 0.2041 validation: 0.950000\n",
            "Iter-37400 loss: 0.2468 validation: 0.950000\n",
            "Iter-37500 loss: 0.1533 validation: 0.960000\n",
            "Iter-37600 loss: 0.2791 validation: 0.950000\n",
            "Iter-37700 loss: 0.2785 validation: 0.960000\n",
            "Iter-37800 loss: 0.1380 validation: 0.960000\n",
            "Iter-37900 loss: 0.2390 validation: 0.950000\n",
            "Iter-38000 loss: 0.1507 validation: 0.960000\n",
            "Iter-38100 loss: 0.3129 validation: 0.950000\n",
            "Iter-38200 loss: 0.2614 validation: 0.960000\n",
            "Iter-38300 loss: 0.2084 validation: 0.970000\n",
            "Iter-38400 loss: 0.2186 validation: 0.960000\n",
            "Iter-38500 loss: 0.1565 validation: 0.970000\n",
            "Iter-38600 loss: 0.1730 validation: 0.970000\n",
            "Iter-38700 loss: 0.2998 validation: 0.960000\n",
            "Iter-38800 loss: 0.2195 validation: 0.970000\n",
            "Iter-38900 loss: 0.3910 validation: 0.960000\n",
            "Iter-39000 loss: 0.2367 validation: 0.960000\n",
            "Iter-39100 loss: 0.1651 validation: 0.960000\n",
            "Iter-39200 loss: 0.2312 validation: 0.960000\n",
            "Iter-39300 loss: 0.2345 validation: 0.970000\n",
            "Iter-39400 loss: 0.2501 validation: 0.970000\n",
            "Iter-39500 loss: 0.1716 validation: 0.970000\n",
            "Iter-39600 loss: 0.1876 validation: 0.960000\n",
            "Iter-39700 loss: 0.1873 validation: 0.970000\n",
            "Iter-39800 loss: 0.1128 validation: 0.970000\n",
            "Iter-39900 loss: 0.1799 validation: 0.960000\n",
            "Iter-40000 loss: 0.2323 validation: 0.960000\n",
            "Iter-40100 loss: 0.2002 validation: 0.960000\n",
            "Iter-40200 loss: 0.2059 validation: 0.960000\n",
            "Iter-40300 loss: 0.1983 validation: 0.970000\n",
            "Iter-40400 loss: 0.5127 validation: 0.970000\n",
            "Iter-40500 loss: 0.1241 validation: 0.980000\n",
            "Iter-40600 loss: 0.2220 validation: 0.970000\n",
            "Iter-40700 loss: 0.2835 validation: 0.970000\n",
            "Iter-40800 loss: 0.1792 validation: 0.970000\n",
            "Iter-40900 loss: 0.1626 validation: 0.970000\n",
            "Iter-41000 loss: 0.1548 validation: 0.970000\n",
            "Iter-41100 loss: 0.2846 validation: 0.960000\n",
            "Iter-41200 loss: 0.1839 validation: 0.970000\n",
            "Iter-41300 loss: 0.1166 validation: 0.970000\n",
            "Iter-41400 loss: 0.2036 validation: 0.970000\n",
            "Iter-41500 loss: 0.1545 validation: 0.980000\n",
            "Iter-41600 loss: 0.1955 validation: 0.970000\n",
            "Iter-41700 loss: 0.3094 validation: 0.970000\n",
            "Iter-41800 loss: 0.2254 validation: 0.970000\n",
            "Iter-41900 loss: 0.1151 validation: 0.970000\n",
            "Iter-42000 loss: 0.1493 validation: 0.970000\n",
            "Iter-42100 loss: 0.1216 validation: 0.960000\n",
            "Iter-42200 loss: 0.1917 validation: 0.970000\n",
            "Iter-42300 loss: 0.3048 validation: 0.970000\n",
            "Iter-42400 loss: 0.3828 validation: 0.970000\n",
            "Iter-42500 loss: 0.1590 validation: 0.970000\n",
            "Iter-42600 loss: 0.1964 validation: 0.980000\n",
            "Iter-42700 loss: 0.1366 validation: 0.970000\n",
            "Iter-42800 loss: 0.1395 validation: 0.970000\n",
            "Iter-42900 loss: 0.2193 validation: 0.970000\n",
            "Iter-43000 loss: 0.1610 validation: 0.970000\n",
            "Iter-43100 loss: 0.1826 validation: 0.970000\n",
            "Iter-43200 loss: 0.2917 validation: 0.970000\n",
            "Iter-43300 loss: 0.1884 validation: 0.970000\n",
            "Iter-43400 loss: 0.1689 validation: 0.970000\n",
            "Iter-43500 loss: 0.2610 validation: 0.970000\n",
            "Iter-43600 loss: 0.1383 validation: 0.970000\n",
            "Iter-43700 loss: 0.1217 validation: 0.980000\n",
            "Iter-43800 loss: 0.1935 validation: 0.970000\n",
            "Iter-43900 loss: 0.2801 validation: 0.970000\n",
            "Iter-44000 loss: 0.1995 validation: 0.970000\n",
            "Iter-44100 loss: 0.1752 validation: 0.960000\n",
            "Iter-44200 loss: 0.1793 validation: 0.970000\n",
            "Iter-44300 loss: 0.1790 validation: 0.970000\n",
            "Iter-44400 loss: 0.2801 validation: 0.970000\n",
            "Iter-44500 loss: 0.1471 validation: 0.970000\n",
            "Iter-44600 loss: 0.3583 validation: 0.970000\n",
            "Iter-44700 loss: 0.3204 validation: 0.970000\n",
            "Iter-44800 loss: 0.1772 validation: 0.970000\n",
            "Iter-44900 loss: 0.2048 validation: 0.970000\n",
            "Iter-45000 loss: 0.2283 validation: 0.970000\n",
            "Iter-45100 loss: 0.3234 validation: 0.970000\n",
            "Iter-45200 loss: 0.5260 validation: 0.970000\n",
            "Iter-45300 loss: 0.1285 validation: 0.970000\n",
            "Iter-45400 loss: 0.1240 validation: 0.970000\n",
            "Iter-45500 loss: 0.1814 validation: 0.970000\n",
            "Iter-45600 loss: 0.1398 validation: 0.970000\n",
            "Iter-45700 loss: 0.1285 validation: 0.980000\n",
            "Iter-45800 loss: 0.1010 validation: 0.980000\n",
            "Iter-45900 loss: 0.2007 validation: 0.970000\n",
            "Iter-46000 loss: 0.1671 validation: 0.960000\n",
            "Iter-46100 loss: 0.2394 validation: 0.960000\n",
            "Iter-46200 loss: 0.1325 validation: 0.970000\n",
            "Iter-46300 loss: 0.1472 validation: 0.970000\n",
            "Iter-46400 loss: 0.0915 validation: 0.970000\n",
            "Iter-46500 loss: 0.2463 validation: 0.970000\n",
            "Iter-46600 loss: 0.2411 validation: 0.970000\n",
            "Iter-46700 loss: 0.1717 validation: 0.970000\n",
            "Iter-46800 loss: 0.2960 validation: 0.980000\n",
            "Iter-46900 loss: 0.2933 validation: 0.970000\n",
            "Iter-47000 loss: 0.2825 validation: 0.970000\n",
            "Iter-47100 loss: 0.2489 validation: 0.970000\n",
            "Iter-47200 loss: 0.2640 validation: 0.980000\n",
            "Iter-47300 loss: 0.1604 validation: 0.980000\n",
            "Iter-47400 loss: 0.1071 validation: 0.980000\n",
            "Iter-47500 loss: 0.1839 validation: 0.970000\n",
            "Iter-47600 loss: 0.2746 validation: 0.980000\n",
            "Iter-47700 loss: 0.2606 validation: 0.980000\n",
            "Iter-47800 loss: 0.2018 validation: 0.970000\n",
            "Iter-47900 loss: 0.2358 validation: 0.970000\n",
            "Iter-48000 loss: 0.1998 validation: 0.980000\n",
            "Iter-48100 loss: 0.2771 validation: 0.980000\n",
            "Iter-48200 loss: 0.3142 validation: 0.970000\n",
            "Iter-48300 loss: 0.1257 validation: 0.980000\n",
            "Iter-48400 loss: 0.1711 validation: 0.980000\n",
            "Iter-48500 loss: 0.1725 validation: 0.980000\n",
            "Iter-48600 loss: 0.1709 validation: 0.970000\n",
            "Iter-48700 loss: 0.1470 validation: 0.970000\n",
            "Iter-48800 loss: 0.1579 validation: 0.980000\n",
            "Iter-48900 loss: 0.1625 validation: 0.980000\n",
            "Iter-49000 loss: 0.1576 validation: 0.980000\n",
            "Iter-49100 loss: 0.2740 validation: 0.980000\n",
            "Iter-49200 loss: 0.1555 validation: 0.980000\n",
            "Iter-49300 loss: 0.2286 validation: 0.980000\n",
            "Iter-49400 loss: 0.1501 validation: 0.980000\n",
            "Iter-49500 loss: 0.2775 validation: 0.980000\n",
            "Iter-49600 loss: 0.0727 validation: 0.980000\n",
            "Iter-49700 loss: 0.1203 validation: 0.980000\n",
            "Iter-49800 loss: 0.1227 validation: 0.980000\n",
            "Iter-49900 loss: 0.2491 validation: 0.980000\n",
            "Iter-50000 loss: 0.1154 validation: 0.980000\n",
            "\n",
            "Best max_norm:  4.0 best dropout rate for input:  0.7 best dropout rate for hidden layers:  0.8 and result accuracy is:  0.9568\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9F7HmF5kyegc"
      },
      "source": [
        "Dropout 3-layer CNN with 128 hidden units\n",
        "(Conv layer with stride=1, padding=1; Maxpool; Fully connected with relu) \n",
        "+ max-norm constraint "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "background_save": true,
          "base_uri": "https://localhost:8080/"
        },
        "id": "AEKreDcaymQM",
        "outputId": "dbaf48b5-2bfa-4a07-d81d-7357f03ef7e8"
      },
      "source": [
        "model = 'cnn'\n",
        "n_iter = 50000\n",
        "alpha = 1e-3 #learning rate\n",
        "mb_size = 64\n",
        "n_experiment = 1\n",
        "reg = 1e-5\n",
        "print_after = 100\n",
        "p_dropout = [0.5,0.8] #only 0.5 is used since only dropout is applied for the only one fully connected layer\n",
        "loss = 'cross_ent'\n",
        "nonlin = 'relu'\n",
        "solver = 'sgd'\n",
        "max_norm = 4\n",
        "nlayer = 3\n",
        "\n",
        "# Define default candidates\n",
        "candidates = np.array([[3, 3.5, 4],[0.5, 0.65, 0.7, 0.8],[0.8]]) # first hyper-param as max-norm, second as drop-out rate for input layer, third as drop-out rate for hidden layers\n",
        "\n",
        "run_mnist(model, n_iter, alpha, mb_size, n_experiment, reg, print_after, p_dropout, loss, nonlin, solver, max_norm, x_train, x_validation, x_test, y_train, y_validation, y_test, candidates, nlayer)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:16: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray\n",
            "  app.launch_new_instance()\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Experimenting on sgd\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  3 input drop out rate:  0.5 hidden layer drop out rate:  0.8\n",
            "Experiment-1\n",
            "Iter-100 loss: 2.2335 validation: 0.580000\n",
            "Iter-200 loss: 2.1804 validation: 0.760000\n",
            "Iter-300 loss: 2.0854 validation: 0.800000\n",
            "Iter-400 loss: 1.9814 validation: 0.830000\n",
            "Iter-500 loss: 1.9085 validation: 0.800000\n",
            "\n",
            "Mean accuracy: 0.7400, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  3 input drop out rate:  0.65 hidden layer drop out rate:  0.8\n",
            "Experiment-1\n",
            "Iter-100 loss: 2.2557 validation: 0.360000\n",
            "Iter-200 loss: 2.2030 validation: 0.510000\n",
            "Iter-300 loss: 2.1085 validation: 0.650000\n",
            "Iter-400 loss: 1.9781 validation: 0.670000\n",
            "Iter-500 loss: 1.9269 validation: 0.730000\n",
            "\n",
            "Mean accuracy: 0.6773, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  3 input drop out rate:  0.7 hidden layer drop out rate:  0.8\n",
            "Experiment-1\n",
            "Iter-100 loss: 2.2461 validation: 0.330000\n",
            "Iter-200 loss: 2.1336 validation: 0.520000\n",
            "Iter-300 loss: 2.0399 validation: 0.690000\n",
            "Iter-400 loss: 1.9442 validation: 0.750000\n",
            "Iter-500 loss: 1.8716 validation: 0.780000\n",
            "\n",
            "Mean accuracy: 0.7305, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  3 input drop out rate:  0.8 hidden layer drop out rate:  0.8\n",
            "Experiment-1\n",
            "Iter-100 loss: 2.2107 validation: 0.380000\n",
            "Iter-200 loss: 2.1358 validation: 0.610000\n",
            "Iter-300 loss: 2.0373 validation: 0.720000\n",
            "Iter-400 loss: 1.9758 validation: 0.750000\n",
            "Iter-500 loss: 1.8116 validation: 0.770000\n",
            "\n",
            "Mean accuracy: 0.7349, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  3.5 input drop out rate:  0.5 hidden layer drop out rate:  0.8\n",
            "Experiment-1\n",
            "Iter-100 loss: 2.1902 validation: 0.520000\n",
            "Iter-200 loss: 2.0720 validation: 0.630000\n",
            "Iter-300 loss: 1.9709 validation: 0.730000\n",
            "Iter-400 loss: 1.7581 validation: 0.760000\n",
            "Iter-500 loss: 1.7402 validation: 0.780000\n",
            "\n",
            "Mean accuracy: 0.7090, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  3.5 input drop out rate:  0.65 hidden layer drop out rate:  0.8\n",
            "Experiment-1\n",
            "Iter-100 loss: 2.2207 validation: 0.460000\n",
            "Iter-200 loss: 2.1225 validation: 0.660000\n",
            "Iter-300 loss: 2.0872 validation: 0.750000\n",
            "Iter-400 loss: 1.9493 validation: 0.770000\n",
            "Iter-500 loss: 1.8060 validation: 0.770000\n",
            "\n",
            "Mean accuracy: 0.7434, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  3.5 input drop out rate:  0.7 hidden layer drop out rate:  0.8\n",
            "Experiment-1\n",
            "Iter-100 loss: 2.2678 validation: 0.330000\n",
            "Iter-200 loss: 2.2237 validation: 0.570000\n",
            "Iter-300 loss: 2.1367 validation: 0.710000\n",
            "Iter-400 loss: 2.0570 validation: 0.770000\n",
            "Iter-500 loss: 1.9752 validation: 0.790000\n",
            "\n",
            "Mean accuracy: 0.6437, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  3.5 input drop out rate:  0.8 hidden layer drop out rate:  0.8\n",
            "Experiment-1\n",
            "Iter-100 loss: 2.2885 validation: 0.420000\n",
            "Iter-200 loss: 2.2467 validation: 0.500000\n",
            "Iter-300 loss: 2.2147 validation: 0.670000\n",
            "Iter-400 loss: 2.1414 validation: 0.740000\n",
            "Iter-500 loss: 2.1208 validation: 0.740000\n",
            "\n",
            "Mean accuracy: 0.6582, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  4 input drop out rate:  0.5 hidden layer drop out rate:  0.8\n",
            "Experiment-1\n",
            "Iter-100 loss: 2.2042 validation: 0.550000\n",
            "Iter-200 loss: 2.1364 validation: 0.720000\n",
            "Iter-300 loss: 1.9461 validation: 0.810000\n",
            "Iter-400 loss: 1.8972 validation: 0.810000\n",
            "Iter-500 loss: 1.7499 validation: 0.810000\n",
            "\n",
            "Mean accuracy: 0.7910, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  4 input drop out rate:  0.65 hidden layer drop out rate:  0.8\n",
            "Experiment-1\n",
            "Iter-100 loss: 2.1211 validation: 0.610000\n",
            "Iter-200 loss: 1.8960 validation: 0.730000\n",
            "Iter-300 loss: 1.6911 validation: 0.840000\n",
            "Iter-400 loss: 1.5808 validation: 0.810000\n",
            "Iter-500 loss: 1.3832 validation: 0.800000\n",
            "\n",
            "Mean accuracy: 0.8010, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  4 input drop out rate:  0.7 hidden layer drop out rate:  0.8\n",
            "Experiment-1\n",
            "Iter-100 loss: 2.1137 validation: 0.660000\n",
            "Iter-200 loss: 1.9369 validation: 0.740000\n",
            "Iter-300 loss: 1.6924 validation: 0.800000\n",
            "Iter-400 loss: 1.5235 validation: 0.810000\n",
            "Iter-500 loss: 1.3747 validation: 0.830000\n",
            "\n",
            "Mean accuracy: 0.7908, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  4 input drop out rate:  0.8 hidden layer drop out rate:  0.8\n",
            "Experiment-1\n",
            "Iter-100 loss: 2.1256 validation: 0.570000\n",
            "Iter-200 loss: 1.8956 validation: 0.680000\n",
            "Iter-300 loss: 1.7461 validation: 0.780000\n",
            "Iter-400 loss: 1.5050 validation: 0.810000\n",
            "Iter-500 loss: 1.3704 validation: 0.820000\n",
            "\n",
            "Mean accuracy: 0.8124, std: 0.0000\n",
            "\n",
            "Experiment-1\n",
            "Iter-100 loss: 2.2609 validation: 0.250000\n",
            "Iter-200 loss: 2.1758 validation: 0.580000\n",
            "Iter-300 loss: 2.0291 validation: 0.710000\n",
            "Iter-400 loss: 1.9590 validation: 0.820000\n",
            "Iter-500 loss: 1.9102 validation: 0.870000\n",
            "Iter-600 loss: 1.7228 validation: 0.880000\n",
            "Iter-700 loss: 1.5987 validation: 0.880000\n",
            "Iter-800 loss: 1.4902 validation: 0.850000\n",
            "Iter-900 loss: 1.3711 validation: 0.850000\n",
            "Iter-1000 loss: 1.2416 validation: 0.840000\n",
            "Iter-1100 loss: 1.1472 validation: 0.870000\n",
            "Iter-1200 loss: 1.1417 validation: 0.860000\n",
            "Iter-1300 loss: 1.0026 validation: 0.850000\n",
            "Iter-1400 loss: 1.0292 validation: 0.870000\n",
            "Iter-1500 loss: 0.9525 validation: 0.860000\n",
            "Iter-1600 loss: 0.7012 validation: 0.870000\n",
            "Iter-1700 loss: 0.7543 validation: 0.880000\n",
            "Iter-1800 loss: 0.8279 validation: 0.880000\n",
            "Iter-1900 loss: 0.9081 validation: 0.890000\n",
            "Iter-2000 loss: 0.7299 validation: 0.890000\n",
            "Iter-2100 loss: 0.7593 validation: 0.890000\n",
            "Iter-2200 loss: 0.5966 validation: 0.890000\n",
            "Iter-2300 loss: 0.7396 validation: 0.880000\n",
            "Iter-2400 loss: 0.7854 validation: 0.890000\n",
            "Iter-2500 loss: 0.6428 validation: 0.880000\n",
            "Iter-2600 loss: 0.5440 validation: 0.890000\n",
            "Iter-2700 loss: 0.6744 validation: 0.910000\n",
            "Iter-2800 loss: 0.6093 validation: 0.910000\n",
            "Iter-2900 loss: 0.4724 validation: 0.900000\n",
            "Iter-3000 loss: 0.5862 validation: 0.910000\n",
            "Iter-3100 loss: 0.4697 validation: 0.910000\n",
            "Iter-3200 loss: 0.6162 validation: 0.910000\n",
            "Iter-3300 loss: 0.6372 validation: 0.920000\n",
            "Iter-3400 loss: 0.5081 validation: 0.920000\n",
            "Iter-3500 loss: 0.6574 validation: 0.920000\n",
            "Iter-3600 loss: 0.6489 validation: 0.920000\n",
            "Iter-3700 loss: 0.5321 validation: 0.920000\n",
            "Iter-3800 loss: 0.4804 validation: 0.920000\n",
            "Iter-3900 loss: 0.5590 validation: 0.920000\n",
            "Iter-4000 loss: 0.5679 validation: 0.920000\n",
            "Iter-4100 loss: 0.5689 validation: 0.920000\n",
            "Iter-4200 loss: 0.4467 validation: 0.930000\n",
            "Iter-4300 loss: 0.4801 validation: 0.930000\n",
            "Iter-4400 loss: 0.4733 validation: 0.930000\n",
            "Iter-4500 loss: 0.4240 validation: 0.930000\n",
            "Iter-4600 loss: 0.5386 validation: 0.930000\n",
            "Iter-4700 loss: 0.4686 validation: 0.930000\n",
            "Iter-4800 loss: 0.4859 validation: 0.930000\n",
            "Iter-4900 loss: 0.6240 validation: 0.930000\n",
            "Iter-5000 loss: 0.4491 validation: 0.930000\n",
            "Iter-5100 loss: 0.4719 validation: 0.930000\n",
            "Iter-5200 loss: 0.4858 validation: 0.930000\n",
            "Iter-5300 loss: 0.3577 validation: 0.930000\n",
            "Iter-5400 loss: 0.4626 validation: 0.930000\n",
            "Iter-5500 loss: 0.4676 validation: 0.930000\n",
            "Iter-5600 loss: 0.5312 validation: 0.930000\n",
            "Iter-5700 loss: 0.6104 validation: 0.930000\n",
            "Iter-5800 loss: 0.4330 validation: 0.930000\n",
            "Iter-5900 loss: 0.3626 validation: 0.930000\n",
            "Iter-6000 loss: 0.4771 validation: 0.930000\n",
            "Iter-6100 loss: 0.3964 validation: 0.930000\n",
            "Iter-6200 loss: 0.3311 validation: 0.930000\n",
            "Iter-6300 loss: 0.4694 validation: 0.930000\n",
            "Iter-6400 loss: 0.6250 validation: 0.930000\n",
            "Iter-6500 loss: 0.4761 validation: 0.930000\n",
            "Iter-6600 loss: 0.3347 validation: 0.930000\n",
            "Iter-6700 loss: 0.4966 validation: 0.930000\n",
            "Iter-6800 loss: 0.3456 validation: 0.930000\n",
            "Iter-6900 loss: 0.3852 validation: 0.930000\n",
            "Iter-7000 loss: 0.7138 validation: 0.930000\n",
            "Iter-7100 loss: 0.3998 validation: 0.930000\n",
            "Iter-7200 loss: 0.5930 validation: 0.930000\n",
            "Iter-7300 loss: 0.3375 validation: 0.930000\n",
            "Iter-7400 loss: 0.3816 validation: 0.930000\n",
            "Iter-7500 loss: 0.2919 validation: 0.930000\n",
            "Iter-7600 loss: 0.4866 validation: 0.930000\n",
            "Iter-7700 loss: 0.3850 validation: 0.930000\n",
            "Iter-7800 loss: 0.4535 validation: 0.930000\n",
            "Iter-7900 loss: 0.3957 validation: 0.930000\n",
            "Iter-8000 loss: 0.3562 validation: 0.930000\n",
            "Iter-8100 loss: 0.2293 validation: 0.930000\n",
            "Iter-8200 loss: 0.3288 validation: 0.930000\n",
            "Iter-8300 loss: 0.2270 validation: 0.930000\n",
            "Iter-8400 loss: 0.5353 validation: 0.930000\n",
            "Iter-8500 loss: 0.5364 validation: 0.930000\n",
            "Iter-8600 loss: 0.3308 validation: 0.930000\n",
            "Iter-8700 loss: 0.3978 validation: 0.930000\n",
            "Iter-8800 loss: 0.3232 validation: 0.930000\n",
            "Iter-8900 loss: 0.3782 validation: 0.930000\n",
            "Iter-9000 loss: 0.2281 validation: 0.930000\n",
            "Iter-9100 loss: 0.3726 validation: 0.930000\n",
            "Iter-9200 loss: 0.3289 validation: 0.930000\n",
            "Iter-9300 loss: 0.2506 validation: 0.930000\n",
            "Iter-9400 loss: 0.3682 validation: 0.930000\n",
            "Iter-9500 loss: 0.6656 validation: 0.930000\n",
            "Iter-9600 loss: 0.3288 validation: 0.930000\n",
            "Iter-9700 loss: 0.3484 validation: 0.930000\n",
            "Iter-9800 loss: 0.3534 validation: 0.930000\n",
            "Iter-9900 loss: 0.5541 validation: 0.930000\n",
            "Iter-10000 loss: 0.3125 validation: 0.930000\n",
            "Iter-10100 loss: 0.3398 validation: 0.930000\n",
            "Iter-10200 loss: 0.3307 validation: 0.930000\n",
            "Iter-10300 loss: 0.4495 validation: 0.930000\n",
            "Iter-10400 loss: 0.3009 validation: 0.930000\n",
            "Iter-10500 loss: 0.2276 validation: 0.930000\n",
            "Iter-10600 loss: 0.3555 validation: 0.930000\n",
            "Iter-10700 loss: 0.3614 validation: 0.930000\n",
            "Iter-10800 loss: 0.2598 validation: 0.930000\n",
            "Iter-10900 loss: 0.3516 validation: 0.930000\n",
            "Iter-11000 loss: 0.3879 validation: 0.930000\n",
            "Iter-11100 loss: 0.2705 validation: 0.930000\n",
            "Iter-11200 loss: 0.4301 validation: 0.930000\n",
            "Iter-11300 loss: 0.4185 validation: 0.930000\n",
            "Iter-11400 loss: 0.1930 validation: 0.930000\n",
            "Iter-11500 loss: 0.3006 validation: 0.930000\n",
            "Iter-11600 loss: 0.4731 validation: 0.930000\n",
            "Iter-11700 loss: 0.3630 validation: 0.930000\n",
            "Iter-11800 loss: 0.2465 validation: 0.930000\n",
            "Iter-11900 loss: 0.4616 validation: 0.930000\n",
            "Iter-12000 loss: 0.3922 validation: 0.930000\n",
            "Iter-12100 loss: 0.4580 validation: 0.930000\n",
            "Iter-12200 loss: 0.4372 validation: 0.930000\n",
            "Iter-12300 loss: 0.3031 validation: 0.930000\n",
            "Iter-12400 loss: 0.2088 validation: 0.930000\n",
            "Iter-12500 loss: 0.4148 validation: 0.930000\n",
            "Iter-12600 loss: 0.2790 validation: 0.930000\n",
            "Iter-12700 loss: 0.3979 validation: 0.930000\n",
            "Iter-12800 loss: 0.3502 validation: 0.930000\n",
            "Iter-12900 loss: 0.4087 validation: 0.930000\n",
            "Iter-13000 loss: 0.3428 validation: 0.930000\n",
            "Iter-13100 loss: 0.2429 validation: 0.930000\n",
            "Iter-13200 loss: 0.4995 validation: 0.930000\n",
            "Iter-13300 loss: 0.3458 validation: 0.930000\n",
            "Iter-13400 loss: 0.3898 validation: 0.930000\n",
            "Iter-13500 loss: 0.2118 validation: 0.930000\n",
            "Iter-13600 loss: 0.5317 validation: 0.930000\n",
            "Iter-13700 loss: 0.3764 validation: 0.930000\n",
            "Iter-13800 loss: 0.4434 validation: 0.930000\n",
            "Iter-13900 loss: 0.3096 validation: 0.930000\n",
            "Iter-14000 loss: 0.3279 validation: 0.930000\n",
            "Iter-14100 loss: 0.4341 validation: 0.930000\n",
            "Iter-14200 loss: 0.4178 validation: 0.940000\n",
            "Iter-14300 loss: 0.2640 validation: 0.940000\n",
            "Iter-14400 loss: 0.2484 validation: 0.930000\n",
            "Iter-14500 loss: 0.2772 validation: 0.930000\n",
            "Iter-14600 loss: 0.2241 validation: 0.940000\n",
            "Iter-14700 loss: 0.4153 validation: 0.930000\n",
            "Iter-14800 loss: 0.4813 validation: 0.930000\n",
            "Iter-14900 loss: 0.3831 validation: 0.940000\n",
            "Iter-15000 loss: 0.1826 validation: 0.940000\n",
            "Iter-15100 loss: 0.2982 validation: 0.940000\n",
            "Iter-15200 loss: 0.2903 validation: 0.940000\n",
            "Iter-15300 loss: 0.2356 validation: 0.940000\n",
            "Iter-15400 loss: 0.3446 validation: 0.940000\n",
            "Iter-15500 loss: 0.3351 validation: 0.940000\n",
            "Iter-15600 loss: 0.2985 validation: 0.940000\n",
            "Iter-15700 loss: 0.2763 validation: 0.940000\n",
            "Iter-15800 loss: 0.2210 validation: 0.940000\n",
            "Iter-15900 loss: 0.3177 validation: 0.940000\n",
            "Iter-16000 loss: 0.2818 validation: 0.940000\n",
            "Iter-16100 loss: 0.3134 validation: 0.940000\n",
            "Iter-16200 loss: 0.3589 validation: 0.940000\n",
            "Iter-16300 loss: 0.4269 validation: 0.940000\n",
            "Iter-16400 loss: 0.2624 validation: 0.940000\n",
            "Iter-16500 loss: 0.3331 validation: 0.940000\n",
            "Iter-16600 loss: 0.3068 validation: 0.940000\n",
            "Iter-16700 loss: 0.4333 validation: 0.940000\n",
            "Iter-16800 loss: 0.2420 validation: 0.940000\n",
            "Iter-16900 loss: 0.5172 validation: 0.940000\n",
            "Iter-17000 loss: 0.2570 validation: 0.940000\n",
            "Iter-17100 loss: 0.3055 validation: 0.940000\n",
            "Iter-17200 loss: 0.4027 validation: 0.940000\n",
            "Iter-17300 loss: 0.2483 validation: 0.940000\n",
            "Iter-17400 loss: 0.2712 validation: 0.940000\n",
            "Iter-17500 loss: 0.2689 validation: 0.940000\n",
            "Iter-17600 loss: 0.2939 validation: 0.940000\n",
            "Iter-17700 loss: 0.4049 validation: 0.940000\n",
            "Iter-17800 loss: 0.3950 validation: 0.940000\n",
            "Iter-17900 loss: 0.2457 validation: 0.940000\n",
            "Iter-18000 loss: 0.2326 validation: 0.940000\n",
            "Iter-18100 loss: 0.2307 validation: 0.940000\n",
            "Iter-18200 loss: 0.3666 validation: 0.950000\n",
            "Iter-18300 loss: 0.3857 validation: 0.940000\n",
            "Iter-18400 loss: 0.3372 validation: 0.940000\n",
            "Iter-18500 loss: 0.2692 validation: 0.940000\n",
            "Iter-18600 loss: 0.2419 validation: 0.940000\n",
            "Iter-18700 loss: 0.4131 validation: 0.950000\n",
            "Iter-18800 loss: 0.3031 validation: 0.940000\n",
            "Iter-18900 loss: 0.3611 validation: 0.940000\n",
            "Iter-19000 loss: 0.2251 validation: 0.940000\n",
            "Iter-19100 loss: 0.2132 validation: 0.940000\n",
            "Iter-19200 loss: 0.2841 validation: 0.950000\n",
            "Iter-19300 loss: 0.1777 validation: 0.940000\n",
            "Iter-19400 loss: 0.3255 validation: 0.940000\n",
            "Iter-19500 loss: 0.5073 validation: 0.950000\n",
            "Iter-19600 loss: 0.3166 validation: 0.940000\n",
            "Iter-19700 loss: 0.2568 validation: 0.950000\n",
            "Iter-19800 loss: 0.3377 validation: 0.950000\n",
            "Iter-19900 loss: 0.4080 validation: 0.940000\n",
            "Iter-20000 loss: 0.3864 validation: 0.940000\n",
            "Iter-20100 loss: 0.2672 validation: 0.950000\n",
            "Iter-20200 loss: 0.1982 validation: 0.940000\n",
            "Iter-20300 loss: 0.1944 validation: 0.940000\n",
            "Iter-20400 loss: 0.4038 validation: 0.940000\n",
            "Iter-20500 loss: 0.3930 validation: 0.950000\n",
            "Iter-20600 loss: 0.3805 validation: 0.950000\n",
            "Iter-20700 loss: 0.1527 validation: 0.940000\n",
            "Iter-20800 loss: 0.2644 validation: 0.950000\n",
            "Iter-20900 loss: 0.3776 validation: 0.960000\n",
            "Iter-21000 loss: 0.4640 validation: 0.950000\n",
            "Iter-21100 loss: 0.2597 validation: 0.950000\n",
            "Iter-21200 loss: 0.2443 validation: 0.960000\n",
            "Iter-21300 loss: 0.3494 validation: 0.960000\n",
            "Iter-21400 loss: 0.4707 validation: 0.960000\n",
            "Iter-21500 loss: 0.2992 validation: 0.960000\n",
            "Iter-21600 loss: 0.1656 validation: 0.960000\n",
            "Iter-21700 loss: 0.1984 validation: 0.960000\n",
            "Iter-21800 loss: 0.2701 validation: 0.960000\n",
            "Iter-21900 loss: 0.2177 validation: 0.960000\n",
            "Iter-22000 loss: 0.2647 validation: 0.960000\n",
            "Iter-22100 loss: 0.4191 validation: 0.960000\n",
            "Iter-22200 loss: 0.3758 validation: 0.960000\n",
            "Iter-22300 loss: 0.1868 validation: 0.960000\n",
            "Iter-22400 loss: 0.2357 validation: 0.960000\n",
            "Iter-22500 loss: 0.3625 validation: 0.970000\n",
            "Iter-22600 loss: 0.2505 validation: 0.960000\n",
            "Iter-22700 loss: 0.2255 validation: 0.970000\n",
            "Iter-22800 loss: 0.3704 validation: 0.960000\n",
            "Iter-22900 loss: 0.3149 validation: 0.960000\n",
            "Iter-23000 loss: 0.3597 validation: 0.960000\n",
            "Iter-23100 loss: 0.4555 validation: 0.960000\n",
            "Iter-23200 loss: 0.3200 validation: 0.960000\n",
            "Iter-23300 loss: 0.2820 validation: 0.960000\n",
            "Iter-23400 loss: 0.2345 validation: 0.970000\n",
            "Iter-23500 loss: 0.3403 validation: 0.960000\n",
            "Iter-23600 loss: 0.3944 validation: 0.960000\n",
            "Iter-23700 loss: 0.2483 validation: 0.960000\n",
            "Iter-23800 loss: 0.3891 validation: 0.960000\n",
            "Iter-23900 loss: 0.2444 validation: 0.960000\n",
            "Iter-24000 loss: 0.2878 validation: 0.960000\n",
            "Iter-24100 loss: 0.1956 validation: 0.960000\n",
            "Iter-24200 loss: 0.2970 validation: 0.960000\n",
            "Iter-24300 loss: 0.2333 validation: 0.960000\n",
            "Iter-24400 loss: 0.2600 validation: 0.960000\n",
            "Iter-24500 loss: 0.2083 validation: 0.960000\n",
            "Iter-24600 loss: 0.2131 validation: 0.960000\n",
            "Iter-24700 loss: 0.1935 validation: 0.960000\n",
            "Iter-24800 loss: 0.3835 validation: 0.970000\n",
            "Iter-24900 loss: 0.1240 validation: 0.960000\n",
            "Iter-25000 loss: 0.2546 validation: 0.960000\n",
            "Iter-25100 loss: 0.2383 validation: 0.960000\n",
            "Iter-25200 loss: 0.2720 validation: 0.960000\n",
            "Iter-25300 loss: 0.1721 validation: 0.960000\n",
            "Iter-25400 loss: 0.2206 validation: 0.960000\n",
            "Iter-25500 loss: 0.2053 validation: 0.960000\n",
            "Iter-25600 loss: 0.3955 validation: 0.960000\n",
            "Iter-25700 loss: 0.2431 validation: 0.960000\n",
            "Iter-25800 loss: 0.3292 validation: 0.960000\n",
            "Iter-25900 loss: 0.3100 validation: 0.960000\n",
            "Iter-26000 loss: 0.2850 validation: 0.960000\n",
            "Iter-26100 loss: 0.1619 validation: 0.960000\n",
            "Iter-26200 loss: 0.2434 validation: 0.960000\n",
            "Iter-26300 loss: 0.2066 validation: 0.960000\n",
            "Iter-26400 loss: 0.5308 validation: 0.960000\n",
            "Iter-26500 loss: 0.1779 validation: 0.960000\n",
            "Iter-26600 loss: 0.1790 validation: 0.960000\n",
            "Iter-26700 loss: 0.2512 validation: 0.970000\n",
            "Iter-26800 loss: 0.4321 validation: 0.970000\n",
            "Iter-26900 loss: 0.3459 validation: 0.970000\n",
            "Iter-27000 loss: 0.2873 validation: 0.970000\n",
            "Iter-27100 loss: 0.2202 validation: 0.970000\n",
            "Iter-27200 loss: 0.1662 validation: 0.970000\n",
            "Iter-27300 loss: 0.2939 validation: 0.970000\n",
            "Iter-27400 loss: 0.3822 validation: 0.970000\n",
            "Iter-27500 loss: 0.2365 validation: 0.970000\n",
            "Iter-27600 loss: 0.2107 validation: 0.970000\n",
            "Iter-27700 loss: 0.1839 validation: 0.970000\n",
            "Iter-27800 loss: 0.2017 validation: 0.970000\n",
            "Iter-27900 loss: 0.1712 validation: 0.970000\n",
            "Iter-28000 loss: 0.5333 validation: 0.970000\n",
            "Iter-28100 loss: 0.3023 validation: 0.970000\n",
            "Iter-28200 loss: 0.2432 validation: 0.970000\n",
            "Iter-28300 loss: 0.3070 validation: 0.970000\n",
            "Iter-28400 loss: 0.2008 validation: 0.970000\n",
            "Iter-28500 loss: 0.2751 validation: 0.970000\n",
            "Iter-28600 loss: 0.2325 validation: 0.970000\n",
            "Iter-28700 loss: 0.2538 validation: 0.970000\n",
            "Iter-28800 loss: 0.2861 validation: 0.970000\n",
            "Iter-28900 loss: 0.2453 validation: 0.970000\n",
            "Iter-29000 loss: 0.2530 validation: 0.970000\n",
            "Iter-29100 loss: 0.4633 validation: 0.970000\n",
            "Iter-29200 loss: 0.2594 validation: 0.960000\n",
            "Iter-29300 loss: 0.2457 validation: 0.970000\n",
            "Iter-29400 loss: 0.3372 validation: 0.970000\n",
            "Iter-29500 loss: 0.2742 validation: 0.960000\n",
            "Iter-29600 loss: 0.2196 validation: 0.960000\n",
            "Iter-29700 loss: 0.2448 validation: 0.960000\n",
            "Iter-29800 loss: 0.1671 validation: 0.960000\n",
            "Iter-29900 loss: 0.2741 validation: 0.960000\n",
            "Iter-30000 loss: 0.2815 validation: 0.960000\n",
            "Iter-30100 loss: 0.1836 validation: 0.970000\n",
            "Iter-30200 loss: 0.1634 validation: 0.960000\n",
            "Iter-30300 loss: 0.1947 validation: 0.970000\n",
            "Iter-30400 loss: 0.2946 validation: 0.970000\n",
            "Iter-30500 loss: 0.2281 validation: 0.970000\n",
            "Iter-30600 loss: 0.1725 validation: 0.970000\n",
            "Iter-30700 loss: 0.2278 validation: 0.970000\n",
            "Iter-30800 loss: 0.2594 validation: 0.970000\n",
            "Iter-30900 loss: 0.3032 validation: 0.970000\n",
            "Iter-31000 loss: 0.2810 validation: 0.960000\n",
            "Iter-31100 loss: 0.4619 validation: 0.970000\n",
            "Iter-31200 loss: 0.2375 validation: 0.970000\n",
            "Iter-31300 loss: 0.2233 validation: 0.970000\n",
            "Iter-31400 loss: 0.1927 validation: 0.970000\n",
            "Iter-31500 loss: 0.1535 validation: 0.970000\n",
            "Iter-31600 loss: 0.4294 validation: 0.970000\n",
            "Iter-31700 loss: 0.2998 validation: 0.970000\n",
            "Iter-31800 loss: 0.2825 validation: 0.970000\n",
            "Iter-31900 loss: 0.2469 validation: 0.970000\n",
            "Iter-32000 loss: 0.2554 validation: 0.970000\n",
            "Iter-32100 loss: 0.1041 validation: 0.970000\n",
            "Iter-32200 loss: 0.2374 validation: 0.970000\n",
            "Iter-32300 loss: 0.2927 validation: 0.970000\n",
            "Iter-32400 loss: 0.3240 validation: 0.970000\n",
            "Iter-32500 loss: 0.3904 validation: 0.970000\n",
            "Iter-32600 loss: 0.2462 validation: 0.970000\n",
            "Iter-32700 loss: 0.3741 validation: 0.970000\n",
            "Iter-32800 loss: 0.3903 validation: 0.970000\n",
            "Iter-32900 loss: 0.2104 validation: 0.960000\n",
            "Iter-33000 loss: 0.2761 validation: 0.970000\n",
            "Iter-33100 loss: 0.2903 validation: 0.960000\n",
            "Iter-33200 loss: 0.2684 validation: 0.970000\n",
            "Iter-33300 loss: 0.2268 validation: 0.970000\n",
            "Iter-33400 loss: 0.2199 validation: 0.960000\n",
            "Iter-33500 loss: 0.1752 validation: 0.970000\n",
            "Iter-33600 loss: 0.2939 validation: 0.960000\n",
            "Iter-33700 loss: 0.1717 validation: 0.970000\n",
            "Iter-33800 loss: 0.2515 validation: 0.970000\n",
            "Iter-33900 loss: 0.2604 validation: 0.970000\n",
            "Iter-34000 loss: 0.2440 validation: 0.970000\n",
            "Iter-34100 loss: 0.1847 validation: 0.970000\n",
            "Iter-34200 loss: 0.2202 validation: 0.970000\n",
            "Iter-34300 loss: 0.1206 validation: 0.960000\n",
            "Iter-34400 loss: 0.1310 validation: 0.960000\n",
            "Iter-34500 loss: 0.2584 validation: 0.970000\n",
            "Iter-34600 loss: 0.3335 validation: 0.960000\n",
            "Iter-34700 loss: 0.1935 validation: 0.970000\n",
            "Iter-34800 loss: 0.3384 validation: 0.970000\n",
            "Iter-34900 loss: 0.1762 validation: 0.970000\n",
            "Iter-35000 loss: 0.1656 validation: 0.970000\n",
            "Iter-35100 loss: 0.2506 validation: 0.970000\n",
            "Iter-35200 loss: 0.1513 validation: 0.970000\n",
            "Iter-35300 loss: 0.3522 validation: 0.970000\n",
            "Iter-35400 loss: 0.3112 validation: 0.970000\n",
            "Iter-35500 loss: 0.1354 validation: 0.970000\n",
            "Iter-35600 loss: 0.2726 validation: 0.970000\n",
            "Iter-35700 loss: 0.1366 validation: 0.970000\n",
            "Iter-35800 loss: 0.2564 validation: 0.970000\n",
            "Iter-35900 loss: 0.2122 validation: 0.970000\n",
            "Iter-36000 loss: 0.4338 validation: 0.970000\n",
            "Iter-36100 loss: 0.3609 validation: 0.970000\n",
            "Iter-36200 loss: 0.1667 validation: 0.970000\n",
            "Iter-36300 loss: 0.3241 validation: 0.970000\n",
            "Iter-36400 loss: 0.2916 validation: 0.970000\n",
            "Iter-36500 loss: 0.2914 validation: 0.970000\n",
            "Iter-36600 loss: 0.1457 validation: 0.970000\n",
            "Iter-36700 loss: 0.2256 validation: 0.970000\n",
            "Iter-36800 loss: 0.2933 validation: 0.970000\n",
            "Iter-36900 loss: 0.4876 validation: 0.970000\n",
            "Iter-37000 loss: 0.1629 validation: 0.970000\n",
            "Iter-37100 loss: 0.2740 validation: 0.970000\n",
            "Iter-37200 loss: 0.1634 validation: 0.970000\n",
            "Iter-37300 loss: 0.1731 validation: 0.970000\n",
            "Iter-37400 loss: 0.1512 validation: 0.980000\n",
            "Iter-37500 loss: 0.2470 validation: 0.970000\n",
            "Iter-37600 loss: 0.2499 validation: 0.970000\n",
            "Iter-37700 loss: 0.2242 validation: 0.970000\n",
            "Iter-37800 loss: 0.2697 validation: 0.970000\n",
            "Iter-37900 loss: 0.3487 validation: 0.960000\n",
            "Iter-38000 loss: 0.2859 validation: 0.970000\n",
            "Iter-38100 loss: 0.2327 validation: 0.970000\n",
            "Iter-38200 loss: 0.2146 validation: 0.970000\n",
            "Iter-38300 loss: 0.2645 validation: 0.970000\n",
            "Iter-38400 loss: 0.2097 validation: 0.980000\n",
            "Iter-38500 loss: 0.2906 validation: 0.970000\n",
            "Iter-38600 loss: 0.2434 validation: 0.970000\n",
            "Iter-38700 loss: 0.2670 validation: 0.970000\n",
            "Iter-38800 loss: 0.2768 validation: 0.970000\n",
            "Iter-38900 loss: 0.3296 validation: 0.980000\n",
            "Iter-39000 loss: 0.2656 validation: 0.970000\n",
            "Iter-39100 loss: 0.3427 validation: 0.980000\n",
            "Iter-39200 loss: 0.1108 validation: 0.970000\n",
            "Iter-39300 loss: 0.1978 validation: 0.970000\n",
            "Iter-39400 loss: 0.2212 validation: 0.970000\n",
            "Iter-39500 loss: 0.3443 validation: 0.970000\n",
            "Iter-39600 loss: 0.1676 validation: 0.970000\n",
            "Iter-39700 loss: 0.2723 validation: 0.970000\n",
            "Iter-39800 loss: 0.2475 validation: 0.970000\n",
            "Iter-39900 loss: 0.3254 validation: 0.980000\n",
            "Iter-40000 loss: 0.2429 validation: 0.970000\n",
            "Iter-40100 loss: 0.2859 validation: 0.970000\n",
            "Iter-40200 loss: 0.1071 validation: 0.970000\n",
            "Iter-40300 loss: 0.2597 validation: 0.970000\n",
            "Iter-40400 loss: 0.1070 validation: 0.970000\n",
            "Iter-40500 loss: 0.2095 validation: 0.970000\n",
            "Iter-40600 loss: 0.1936 validation: 0.970000\n",
            "Iter-40700 loss: 0.2387 validation: 0.970000\n",
            "Iter-40800 loss: 0.2867 validation: 0.970000\n",
            "Iter-40900 loss: 0.2539 validation: 0.970000\n",
            "Iter-41000 loss: 0.3960 validation: 0.980000\n",
            "Iter-41100 loss: 0.3261 validation: 0.970000\n",
            "Iter-41200 loss: 0.4656 validation: 0.970000\n",
            "Iter-41300 loss: 0.2310 validation: 0.980000\n",
            "Iter-41400 loss: 0.1912 validation: 0.980000\n",
            "Iter-41500 loss: 0.3067 validation: 0.970000\n",
            "Iter-41600 loss: 0.1905 validation: 0.970000\n",
            "Iter-41700 loss: 0.2373 validation: 0.980000\n",
            "Iter-41800 loss: 0.2485 validation: 0.980000\n",
            "Iter-41900 loss: 0.2988 validation: 0.970000\n",
            "Iter-42000 loss: 0.1328 validation: 0.970000\n",
            "Iter-42100 loss: 0.1595 validation: 0.970000\n",
            "Iter-42200 loss: 0.1924 validation: 0.970000\n",
            "Iter-42300 loss: 0.3310 validation: 0.970000\n",
            "Iter-42400 loss: 0.2256 validation: 0.980000\n",
            "Iter-42500 loss: 0.1812 validation: 0.980000\n",
            "Iter-42600 loss: 0.2558 validation: 0.980000\n",
            "Iter-42700 loss: 0.2296 validation: 0.980000\n",
            "Iter-42800 loss: 0.1511 validation: 0.970000\n",
            "Iter-42900 loss: 0.2982 validation: 0.980000\n",
            "Iter-43000 loss: 0.1628 validation: 0.980000\n",
            "Iter-43100 loss: 0.2257 validation: 0.980000\n",
            "Iter-43200 loss: 0.2959 validation: 0.970000\n",
            "Iter-43300 loss: 0.1814 validation: 0.980000\n",
            "Iter-43400 loss: 0.2861 validation: 0.980000\n",
            "Iter-43500 loss: 0.2064 validation: 0.980000\n",
            "Iter-43600 loss: 0.2779 validation: 0.970000\n",
            "Iter-43700 loss: 0.2504 validation: 0.980000\n",
            "Iter-43800 loss: 0.2602 validation: 0.980000\n",
            "Iter-43900 loss: 0.1944 validation: 0.980000\n",
            "Iter-44000 loss: 0.2670 validation: 0.980000\n",
            "Iter-44100 loss: 0.2254 validation: 0.980000\n",
            "Iter-44200 loss: 0.2836 validation: 0.980000\n",
            "Iter-44300 loss: 0.3190 validation: 0.980000\n",
            "Iter-44400 loss: 0.1341 validation: 0.980000\n",
            "Iter-44500 loss: 0.1639 validation: 0.980000\n",
            "Iter-44600 loss: 0.1704 validation: 0.980000\n",
            "Iter-44700 loss: 0.1373 validation: 0.980000\n",
            "Iter-44800 loss: 0.3358 validation: 0.980000\n",
            "Iter-44900 loss: 0.2246 validation: 0.980000\n",
            "Iter-45000 loss: 0.2064 validation: 0.980000\n",
            "Iter-45100 loss: 0.3652 validation: 0.980000\n",
            "Iter-45200 loss: 0.1227 validation: 0.980000\n",
            "Iter-45300 loss: 0.1898 validation: 0.980000\n",
            "Iter-45400 loss: 0.1202 validation: 0.980000\n",
            "Iter-45500 loss: 0.4189 validation: 0.980000\n",
            "Iter-45600 loss: 0.1464 validation: 0.980000\n",
            "Iter-45700 loss: 0.2388 validation: 0.980000\n",
            "Iter-45800 loss: 0.2870 validation: 0.980000\n",
            "Iter-45900 loss: 0.2688 validation: 0.980000\n",
            "Iter-46000 loss: 0.1289 validation: 0.980000\n",
            "Iter-46100 loss: 0.2447 validation: 0.980000\n",
            "Iter-46200 loss: 0.1461 validation: 0.980000\n",
            "Iter-46300 loss: 0.3140 validation: 0.980000\n",
            "Iter-46400 loss: 0.2544 validation: 0.980000\n",
            "Iter-46500 loss: 0.1289 validation: 0.980000\n",
            "Iter-46600 loss: 0.2049 validation: 0.980000\n",
            "Iter-46700 loss: 0.3743 validation: 0.980000\n",
            "Iter-46800 loss: 0.1864 validation: 0.980000\n",
            "Iter-46900 loss: 0.2689 validation: 0.980000\n",
            "Iter-47000 loss: 0.3990 validation: 0.980000\n",
            "Iter-47100 loss: 0.2851 validation: 0.980000\n",
            "Iter-47200 loss: 0.3847 validation: 0.980000\n",
            "Iter-47300 loss: 0.1245 validation: 0.980000\n",
            "Iter-47400 loss: 0.2442 validation: 0.980000\n",
            "Iter-47500 loss: 0.2006 validation: 0.980000\n",
            "Iter-47600 loss: 0.2993 validation: 0.980000\n",
            "Iter-47700 loss: 0.1723 validation: 0.980000\n",
            "Iter-47800 loss: 0.1886 validation: 0.980000\n",
            "Iter-47900 loss: 0.1774 validation: 0.980000\n",
            "Iter-48000 loss: 0.4578 validation: 0.980000\n",
            "Iter-48100 loss: 0.2589 validation: 0.980000\n",
            "Iter-48200 loss: 0.2238 validation: 0.980000\n",
            "Iter-48300 loss: 0.1695 validation: 0.980000\n",
            "Iter-48400 loss: 0.1005 validation: 0.980000\n",
            "Iter-48500 loss: 0.2095 validation: 0.980000\n",
            "Iter-48600 loss: 0.1849 validation: 0.980000\n",
            "Iter-48700 loss: 0.1517 validation: 0.980000\n",
            "Iter-48800 loss: 0.1348 validation: 0.980000\n",
            "Iter-48900 loss: 0.1896 validation: 0.980000\n",
            "Iter-49000 loss: 0.2336 validation: 0.980000\n",
            "Iter-49100 loss: 0.1898 validation: 0.980000\n",
            "Iter-49200 loss: 0.1149 validation: 0.980000\n",
            "Iter-49300 loss: 0.2018 validation: 0.980000\n",
            "Iter-49400 loss: 0.2560 validation: 0.980000\n",
            "Iter-49500 loss: 0.2262 validation: 0.980000\n",
            "Iter-49600 loss: 0.2587 validation: 0.980000\n",
            "Iter-49700 loss: 0.1106 validation: 0.980000\n",
            "Iter-49800 loss: 0.3056 validation: 0.980000\n",
            "Iter-49900 loss: 0.2608 validation: 0.980000\n",
            "Iter-50000 loss: 0.1296 validation: 0.980000\n",
            "\n",
            "Best max_norm:  4.0 best dropout rate for input:  0.8 best dropout rate for hidden layers:  0.8 and result accuracy is:  0.9526\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}