reproduce_models1.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "reproduce_models1.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tgGheJCNKKMi",
        "outputId": "a02f1013-2c2a-47a2-a81e-62a9ed5f91b6"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive', force_remount=True)\n",
        "import sys\n",
        "#sys.path.append('/content/gdrive/hipsternet-master')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qCu02wlCPYnd"
      },
      "source": [
        "sys.path.insert(0,'/content/drive/My Drive/hipsternet-master')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XN3bBYMlN5pZ"
      },
      "source": [
        "import hipsternet"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "a9kb53pqP3zh"
      },
      "source": [
        "import numpy as np\n",
        "import hipsternet.input_data as input_data\n",
        "import hipsternet.neuralnet as nn\n",
        "from hipsternet.solver import *"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RXvo4y6dk_T5"
      },
      "source": [
        "import tensorflow as tf\n",
        "import sympy\n",
        "import numpy as np\n",
        "import seaborn as sns\n",
        "import collections\n",
        "import torch\n",
        "from sklearn.model_selection import GridSearchCV\n",
        "from sklearn.metrics import make_scorer\n",
        "from __future__ import print_function\n",
        "\n",
        "np.random.seed(123)\n",
        "\n",
        "# visualization tools\n",
        "%matplotlib inline\n",
        "import matplotlib.pyplot as plt"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dsz5TNAzlC8t",
        "outputId": "1c41f623-78b2-4549-ea9a-94e1f34aaeb0"
      },
      "source": [
        "(x_train, y_train), (x_test, y_test) = tf.keras.datasets.mnist.load_data()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/mnist.npz\n",
            "11493376/11490434 [==============================] - 0s 0us/step\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 549
        },
        "id": "sBqsQBIoazGU",
        "outputId": "2f3a3673-aaea-43f5-e061-5cc86603a374"
      },
      "source": [
        "import tensorflow_datasets as tfds \n",
        "from keras.preprocessing.image import ImageDataGenerator\n",
        "\n",
        "#Generating rotated images: rotated 15%, fill empty pixal with closest color\n",
        "datagen = ImageDataGenerator(rotation_range=15,horizontal_flip=False,fill_mode='nearest')\n",
        "datagen.fit(x_train.reshape(x_train.shape[0],28,28,1))\n",
        "\n",
        "#Print the first rotated image\n",
        "print('Before rotating 15%')\n",
        "plt.imshow(x_train[0])\n",
        "plt.colorbar()\n",
        "plt.show()\n",
        "\n",
        "print('After rotating 15%')\n",
        "for X, Y in datagen.flow(x_train.reshape(x_train.shape[0], 28, 28, 1),y_train.reshape(y_train.shape[0], 1),batch_size=32,shuffle=False):\n",
        "        plt.imshow(X[0].reshape(28,28))\n",
        "        plt.colorbar()\n",
        "        plt.show()\n",
        "        break\n",
        "\n",
        "#append new images to training set: image size should be at least 3D, so reshape is necessary here\n",
        "i=0\n",
        "x_train_aug = x_train\n",
        "y_train_aug = y_train\n",
        "for X, Y in datagen.flow(x_train.reshape(x_train.shape[0], 28, 28, 1),y_train.reshape(y_train.shape[0], 1),batch_size=32,shuffle=False):\n",
        "        x_train_aug = np.vstack([x_train_aug, X.reshape(X.shape[0],28,28)])\n",
        "        y_train_aug = np.append(y_train_aug, Y)\n",
        "        i += 1\n",
        "        if (i > x_train.shape[0]/32): break #fix here"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Before rotating 15%\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAATEAAAD4CAYAAACE9dGgAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAW7ElEQVR4nO3dfbBd1VnH8e/PEIIEtIlpYwppgxhsA7WhXmkcGEoHpZTpTGC0FHTaWNFgS2xR1NKMY1HLDDqFSmtlvEgkzEApLSDRwVKa6fRFS2iIKSSklEiDEC83DSmQvkFy7uMfZ99ybs496+x7z9teN78Ps+ees5/9suYwPKy19lprKyIwM8vVTw26AGZmnXASM7OsOYmZWdacxMwsa05iZpa1I/p5syM1J45ibj9vaXZY+TE/4KV4UZ1c421vnRvP7quVOvahh1+8LyLO7eR+neooiUk6F7gemAX8c0Rckzr+KObyZp3dyS3NLGFTbOz4Gs/uq/Hgfa8pdeysRY8v6PiGHZp2c1LSLOBTwNuBZcDFkpZ1q2BmNhgBjJX8px1JiyV9SdKjkrZL+mCx/ypJuyVtLbbzGs75sKSdkh6T9LZ29+ikJnYasDMinihufDuwEni0g2ua2YAFwYEo15ws4SBwRURskXQs8JCk+4vYxyPiY40HFxWhi4CTgVcDX5R0UkTrAnXSsX8c8FTD96eLfRNIWi1ps6TNB3ixg9uZWb90qyYWESMRsaX4vB/YwSR5osFK4PaIeDEivgPspF5haqnnTycjYjgihiJiaDZzen07M+tQENSi3AYsGK+kFNvqVteVtAQ4FdhU7Foj6WFJ6yTNK/aVqhw16iSJ7QYWN3w/vthnZpkbI0ptwN7xSkqxDU92PUnHAHcCl0fEC8ANwInAcmAEuHa6Ze0kiX0DWCrpBElHUm/HbujgemZWAQHUiFJbGZJmU09gt0bEXQARMRoRtYgYA27k5SbjlCtH005iEXEQWAPcR72de0dEbJ/u9cysOqZQE0uSJOAmYEdEXNewf1HDYRcA24rPG4CLJM2RdAKwFHgwdY+OxolFxL3AvZ1cw8yqJYAD3Vui63Tg3cAjkrYW+9ZSH5K1vLjdLuBSgIjYLukO6qMcDgKXpZ5MQp9H7JtZ9cUUmoptrxXxNWCyGQQtKz8RcTVwddl7OImZ2UQBtYzWSnUSM7MJ6iP28+EkZmaHELVJW4DV5CRmZhPUO/adxMwsU/VxYk5iZpaxMdfEzCxXromZWdYCUcto5XonMTNr4uakmWUrEC/FrEEXozQnMTOboD7Y1c1JM8uYO/bNLFsRohauiZlZxsZcEzOzXNU79vNJDfmU1Mz6wh37Zpa9mseJmVmuPGLfzLI35qeTZpar+gRwJzEzy1QgDnjakZnlKgIPdjWznMmDXc0sX4FrYmaWOXfsm1m2AnlRRDPLV/2VbfmkhnxKamZ94pfnWoXoiPS/4lmvXNDT+z/2p0taxmpHjyXPfe2Je5Lxo9+f/g/tmeuObBnbMvSZ5Ll7az9Ixt/82SuS8V/8kweS8SoLDqMR+5J2AfuBGnAwIoa6USgzG6zDrSb21ojY24XrmFkFROjwqYmZ2cxT79g/fKYdBfAFSQH8U0QMH3qApNXAaoCjOLrD25lZ7+W1xn6nJT0jIt4EvB24TNKZhx4QEcMRMRQRQ7OZ0+HtzKzX6h37KrW1I2mxpC9JelTSdkkfLPbPl3S/pMeLv/OK/ZL0CUk7JT0s6U3t7tFREouI3cXfPcDdwGmdXM/MqqHGT5XaSjgIXBERy4AV1Cs7y4ArgY0RsRTYWHyHeoVoabGtBm5od4NpJzFJcyUdO/4ZOAfYNt3rmVk1jI/Y70ZNLCJGImJL8Xk/sAM4DlgJrC8OWw+cX3xeCdwSdQ8Ar5C0KHWPTvrEFgJ3Sxq/zm0R8fkOrjdjzXr90mQ85sxOxv/vLa9Ixn+0ovWYpvk/mx7v9NU3psdLDdJ//PDYZPxv/+HcZHzTG25rGfvOgR8lz71m9DeS8Vd/NZLx3E3hRSELJG1u+D48Wd84gKQlwKnAJmBhRIwUoWeo5xOoJ7inGk57utg3QgvTTmIR8QTwxumeb2bVFAEHxkonsb1lxodKOga4E7g8Il4oKj/F/SKKh4PT4iEWZjZBvTnZvaeTkmZTT2C3RsRdxe5RSYsiYqRoLo5Pz9gNLG44/fhiX0v5PEc1s76pFfMn223tqF7lugnYERHXNYQ2AKuKz6uAexr2v6d4SrkCeL6h2Tkp18TMbILxIRZdcjrwbuARSVuLfWuBa4A7JF0CPAlcWMTuBc4DdgI/BN7b7gZOYmZ2iO41JyPia9Cyynb2JMcHcNlU7uEkZmZNvMb+YaZ2VnpQ8XU3fyoZP2l26yVjZrIDUUvG//KTv5uMH/GD9AOtX/vsmpaxY3cfTJ47Z296CMbRmzcl4zmrP508fOZOmtkM4+WpzSx7bk6aWba6/HSy55zEzKyJF0U0s2xFiINOYmaWMzcnzSxb7hM7DM157P+S8Yd+vDgZP2n2aDeL01VXjKxIxp/4fvqVbzef+LmWsefH0uO8Fn7iv5LxXprZC+205yRmZtnyODEzy57HiZlZtiLgYPlFEQfOSczMmrg5aWbZcp+YmWUvnMTMLGfu2D/MHBx5Jhn/5N++Mxm/+tz0a9VmPXxMMv7N938yGU/56N5fTsZ3/vrRyXjtueTy5/z2r72/ZWzXB5KncgLfTB9gPRHhPjEzy5qo+emkmeXMfWJmli3PnTSzvEW9XywXTmJm1sRPJ80sW+GOfTPLnZuTNsH8f/l6Mv7Kf/u5ZLz27L5k/ORTfq9lbPuZ65Lnbhh+SzL+quc6W9NLX2891uuE9M9iA5TT08m2dUZJ6yTtkbStYd98SfdLerz4O6+3xTSzfomoJ7EyWxWUafjeDJx7yL4rgY0RsRTYWHw3sxliLFRqq4K2SSwivgIc2p5ZCawvPq8Hzu9yucxsgCLKbVUw3T6xhRExPmnuGWBhqwMlrQZWAxxFeh6emQ1eIMYyejrZcUkjIki8VyEihiNiKCKGZjOn09uZWR9Eya0KppvERiUtAij+7ulekcxsoGZgx/5kNgCris+rgHu6Uxwzq4SMqmJt+8QkfRo4C1gg6WngI8A1wB2SLgGeBC7sZSFnutreZzs6/8ALR0773JN/59Fk/Ls3zEpfYKw27XtbdVWlllVG2yQWERe3CJ3d5bKYWQUEMDbWnSQmaR3wDmBPRJxS7LsK+APgu8VhayPi3iL2YeASoAZ8ICLua3ePfB5BmFl/BBAqt7V3M83jTAE+HhHLi208gS0DLgJOLs75R0ltmgJOYmY2iW6NE2sxzrSVlcDtEfFiRHwH2Amc1u4kJzEza1a+Y3+BpM0N2+qSd1gj6eFiWuP4tMXjgKcajnm62JfkCeBmdogpDZ/YGxFDU7zBDcDfUE+DfwNcC7RexaAN18TMrFkPh1hExGhE1CJiDLiRl5uMu4HFDYceX+xLck1sBnj9h77dMvbeN6QfIv/Lazcm429552XJ+LGfeSAZtwwFRJeeTk5G0qKGaYsXAOMr5GwAbpN0HfBqYCnwYLvrOYmZ2SS6NsRisnGmZ0laTr0utwu4FCAitku6A3gUOAhcFhFtByI6iZlZsy6Nxm8xzvSmxPFXA1dP5R5OYmbWrCJTispwEjOzicYHu2bCSczMmlRlwcMynMTMrFkPn052m5OYmTWRa2LWT7Xnnm8Ze/Z9r0+e+78bfpSMX/nRW5LxD194QTIe//2zLWOLr27zzrac2jQzSYXWCivDSczMDlF6hYpKcBIzs2auiZlZ1sYGXYDynMTMbCKPEzOz3PnppJnlLaMk5vXEzCxrronNcGPf3JGMX/RXf5aM3/qRjyXjW1ekx5GxonXo5LlrkqcuvXEkGT/4xK70vW3a3Jw0s3wFnnZkZplzTczMcubmpJnlzUnMzLLmJGZmuVK4OWlmufPTScvF/HXpNb3WPJZ+7+TPXPN0Mv7pX7ivZWz7e/4hee7rFv9+Mv5Lf5Ueq117/Ilk3FrLqSbWdsS+pHWS9kja1rDvKkm7JW0ttvN6W0wz66sevgG828pMO7oZOHeS/R+PiOXFdm93i2VmAxMv94u126qgbRKLiK8A+/pQFjOrihlWE2tljaSHi+bmvFYHSVotabOkzQd4sYPbmVm/aKzcVgXTTWI3ACcCy4ER4NpWB0bEcEQMRcTQbOZM83ZmZpObVhKLiNGIqEXEGHAjcFp3i2VmAzXTm5OSFjV8vQDY1upYM8tMZh37bceJSfo0cBawQNLTwEeAsyQtp56LdwGX9rCMNkD6z63J+A9/61XJ+K++649axjZ96Prkud966z8n47+z5Jxk/PkzkmFLqUiCKqNtEouIiyfZfVMPymJmVTGTkpiZHV5EdZ48luEkZmYTVai/qwy/KMTMmnXp6WSLaYvzJd0v6fHi77xivyR9QtLOYgzqm8oU1UnMzJp1b4jFzTRPW7wS2BgRS4GNxXeAtwNLi2019fGobTmJmVmTbg2xaDFtcSWwvvi8Hji/Yf8tUfcA8IpDhnNNyn1i1pHa6J5kfOEnWsd//OcHk+cerSOT8RuX/Hsy/o4LLm997bs3Jc897PW2T2xhRIy/j+8ZYGHx+TjgqYbjni72Jd/d5yRmZhPFlJ5OLpC0ueH7cEQMl75VREidPUZwEjOzZuXTyt6IGJri1UclLYqIkaK5OF5d3w0sbjju+GJfkvvEzKxJj6cdbQBWFZ9XAfc07H9P8ZRyBfB8Q7OzJdfEzKxZl/rEWkxbvAa4Q9IlwJPAhcXh9wLnATuBHwLvLXMPJzEzm6iLK1S0mLYIcPYkxwaQfqnDJJzEzGwCkdeIfScxM2viJGYzxtgZy5Px/3nnUcn4Kct3tYy1GwfWzif3nZqMH33P5mTcEpzEzCxrTmJmlq3MVrFwEjOzZk5iZpYzL4poZllzc9LM8lWh17GV4SRmZs2cxKwqNHRKMv7tD7RZs+v09cn4mUe9NOUylfViHEjGH9h3QvoCY23nDtskPGLfzLKnsXyymJOYmU3kPjEzy52bk2aWNycxM8uZa2JmljcnMTPL1tTedjRwTmIZOOKE1ybj//PeV7eMXfWu25Pn/uYxe6dVpm5YO5p+Sc6Xr1+RjM9b//VuFscKuY0Ta/u2I0mLJX1J0qOStkv6YLF/vqT7JT1e/J3X++KaWV9ElNsqoMwr2w4CV0TEMmAFcJmkZcCVwMaIWApsLL6b2QzQ41e2dVXbJBYRIxGxpfi8H9hB/dXiK4HxOSnrgfN7VUgz66OYwlYBU+oTk7QEOBXYBCxseLHlM8DCFuesBlYDHMXR0y2nmfXRjOzYl3QMcCdweUS8IOknsYgIafLKZUQMA8MAP6P5FcndZpaSUxIr0yeGpNnUE9itEXFXsXtU0qIivgjY05simllfBVl17Letiale5boJ2BER1zWENgCrqL+SfBVwT09KOAMcseQ1yfjzv7IoGX/XX38+Gf/DV9yVjPfSFSPpYRBf/8fWwyjm3/xg8tx5Yx5CMShV6bQvo0xz8nTg3cAjkrYW+9ZST153SLoEeBK4sDdFNLO+m0lJLCK+Rn3822TO7m5xzGzQchvs6hH7ZjZRhBdFNLPM5ZPDnMTMrJmbk2aWrwDcnDSzrOWTw5zEyjpi0c+3jO1bNzd57vtO+HIyfvGxo9MqUzes2X1GMr7lhuXJ+ILPbUvG5+/3WK8cuTlpZlnr5tNJSbuA/UANOBgRQ5LmA58BlgC7gAsj4nvTuX6paUdmdhjpzSoWb42I5RExPoWja0t5OYmZ2QT1wa5RautA15bychIzs2ZjJTdYIGlzw7Z6kqsF8AVJDzXESy3lVYb7xMysyRRqWXsbmoitnBERuyW9Crhf0rcag6mlvMpwTczMJupyn1hE7C7+7gHuBk6ji0t5OYmZ2SHqcyfLbO1Imivp2PHPwDnANl5eygs6XMrrsGlOvvS2dI33pT/el4yv/cV7W8bO+ekfTKtM3TJa+1HL2Jkbrkie+7q/+FYyPv+59DivjBYAtano3oKHC4G7i5WgjwBui4jPS/oGXVrK67BJYmZWUhdfnhsRTwBvnGT/s3RpKS8nMTNrVpGlp8twEjOzZvnkMCcxM2umsXx6O53EzGyiIKsnNk5iZjaB6HhKUV85iZlZMyex6tl1fnpc77ff8Nme3ftTz52YjF//5XOScdVavWyq7nUf/U7L2NLRTclza8moHbacxMwsW+4TM7Pc+emkmWUs3Jw0s4wFTmJmlrl8WpNOYmbWzOPEzCxvMymJSVoM3EJ9XaAAhiPieklXAX8AfLc4dG1EtF50a8BOet+Dyfg73vcrfSpJs5NIl60dj/WyroqAWj7tyTI1sYPAFRGxpVih8SFJ9xexj0fEx3pXPDMbiJlUEyveSDJSfN4vaQdwXK8LZmYDlFESm9Ia+5KWAKcC43NZ1kh6WNI6SfNanLN6/HVOB3ixo8KaWR8EMBbltgooncQkHQPcCVweES8ANwAnAsup19Suney8iBiOiKGIGJrNnC4U2cx6KyDGym0VUOrppKTZ1BPYrRFxF0BEjDbEbwT+vSclNLP+CrLq2G9bE1P9NSU3ATsi4rqG/YsaDruA+muYzGwmiCi3VUCZmtjpwLuBRyRtLfatBS6WtJx63t4FXNqTEppZ/1UkQZVR5unk14DJFrSq7JgwM+tEdWpZZXjEvplNFICX4jGzrLkmZmb5mnnTjszscBIQFRkDVoaTmJk1q8ho/DKcxMysmfvEzCxbEX46aWaZc03MzPIVRC2fpTadxMxsovGleDLhJGZmzTIaYjGlRRHNbOYLIMai1FaGpHMlPSZpp6Qru11eJzEzmyi6tyiipFnAp4C3A8uor36zrJvFdXPSzJp0sWP/NGBnRDwBIOl2YCXwaLdu0Ncktp/v7f1ifO7Jhl0LgL39LMMUVLVsVS0XuGzT1c2yvbbTC+zne/d9MT63oOThR0na3PB9OCKGG74fBzzV8P1p4M2dlrFRX5NYRLyy8bukzREx1M8ylFXVslW1XOCyTVfVyhYR5w66DFPhPjEz66XdwOKG78cX+7rGSczMeukbwFJJJ0g6ErgI2NDNGwy6Y3+4/SEDU9WyVbVc4LJNV5XL1pGIOChpDXAfMAtYFxHbu3kPRUZzpMzMDuXmpJllzUnMzLI2kCTW62kInZC0S9IjkrYeMv5lEGVZJ2mPpG0N++ZLul/S48XfeRUq21WSdhe/3VZJ5w2obIslfUnSo5K2S/pgsX+gv12iXJX43XLV9z6xYhrCt4HfoD7w7RvAxRHRtRG8nZC0CxiKiIEPjJR0JvB94JaIOKXY93fAvoi4pvgfwLyI+FBFynYV8P2I+Fi/y3NI2RYBiyJii6RjgYeA84HfZYC/XaJcF1KB3y1Xg6iJ/WQaQkS8BIxPQ7BDRMRXgH2H7F4JrC8+r6f+H0HftShbJUTESERsKT7vB3ZQHzk+0N8uUS7rwCCS2GTTEKr0LzKAL0h6SNLqQRdmEgsjYqT4/AywcJCFmcQaSQ8Xzc2BNHUbSVoCnApsokK/3SHlgor9bjlxx36zMyLiTdRn3V9WNJsqKep9AVUaI3MDcCKwHBgBrh1kYSQdA9wJXB4RLzTGBvnbTVKuSv1uuRlEEuv5NIRORMTu4u8e4G7qzd8qGS36Vsb7WPYMuDw/ERGjEVGL+ksLb2SAv52k2dQTxa0RcVexe+C/3WTlqtLvlqNBJLGeT0OYLklziw5XJM0FzgG2pc/quw3AquLzKuCeAZZlgvEEUbiAAf12kgTcBOyIiOsaQgP97VqVqyq/W64GMmK/eIT897w8DeHqvhdiEpJ+gXrtC+pTsm4bZNkkfRo4i/pSLaPAR4B/Be4AXgM8CVwYEX3vYG9RtrOoN4kC2AVc2tAH1c+ynQF8FXgEGF+5by31/qeB/XaJcl1MBX63XHnakZllzR37ZpY1JzEzy5qTmJllzUnMzLLmJGZmWXMSM7OsOYmZWdb+H71dPbjcA328AAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        },
        {
          "output_type": "stream",
          "text": [
            "After rotating 15%\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAATEAAAD6CAYAAADJPXCrAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAYUUlEQVR4nO3dbZAd1X3n8e+P0SBhSQ4CxfIgDQiDYq/sxIJMgMQswWFtY9a1gq0ygaSwksURW4EqU3FSIbyI8QuqKMfgtcsOtcJoEVXYmApgtF5iDLID4cEYCfOgB7zIWDISIylCBI3ASJqZf17cVnLn4Z7bM/fO3Huufh+qa/qef/ftox7pz+nTp08rIjAzy9Uxra6AmVkjnMTMLGtOYmaWNScxM8uak5iZZc1JzMyy5iRmZlNGUq+kH0naLGmTpM8V5TdI2inpuWK5qGqfv5G0VdLPJH2i7jGmc5zYsZoZs5g9bcczO9q8w1scioNq5Ds+8dHZ8fq+oVLbbnjh4EMRcWGtuKQeoCcinpU0F9gAXAxcChyIiC+P2n4p8G3gLOAk4BHgNyKiZoVmlKpp7QpeCHwV6AK+GRE3pbafxWzO1gWNHNLMEp6OdQ1/x+v7hvjJQyeX2rar5+X5qXhE9AP9xfqApC3AwsQuy4G7I+Ig8AtJW6kktKdq7TDpy0lJXcA3gE8CS4HLiyxqZhkLYLjkfxMhaTFwBvB0UXSNpBckrZY0ryhbCLxatdsO0kmvoT6xs4CtEfFKRBwC7qaSRc0sY0FwOIZKLcB8SeurlpXjfaekOcC9wLURsR+4FTgNWEalpXbzZOvbyOXkeBnz7NEbFX+olQCzeFcDhzOz6TKBVtbeiOhLbSCpm0oCuysi7gOIiN1V8duA7xUfdwK9VbsvKspqmvK7kxGxKiL6IqKvm5lTfTgza1AQDEW5pR5JAm4HtkTELVXlPVWbXQJsLNbXApdJminpVGAJ8JPUMRppiU04Y5pZHoZp2qiFjwBXAC9Keq4ou55KH/oyKl1w24CrACJik6R7gM3AIHB16s4kNJbEngGWFNlyJ3AZ8EcNfJ+ZtYEAhpqUxCLicWC8IR8PJva5Ebix7DEmncQiYlDSNcBDVIZYrI6ITZP9PjNrH01siU25hsaJRcSDJDKqmeUngMMZTZbaUBIzs84TRNMuJ6eDk5iZjRQwlE8OcxIzs5EqI/bz4SRmZqOIoXFvKLYnJzEzG6HSse8kZmaZqowTcxIzs4wNuyVmZrlyS8zMshaIoYxmrncSM7MxfDlpZtkKxKHoanU1SnMSM7MRKoNdfTlpZhlzx76ZZStCDIVbYmaWsWG3xMwsV5WO/XxSQz41NbNp4Y59M8vekMeJmVmuPGLfzLI37LuTZparygPgTmLWKVSnbySjt+JYOYE47MeOzCxXEXiwq5nlTB7samb5CtwSM7PMuWPfzLIVyJMimlm+Kq9syyc15FNTM5smR9HLcyVtAwaAIWAwIvqaUamOU2es1Yz3Lkjv35UeszPUc0Lt2LvSv+KuA4eS8ehOH3vG9j3JeGoc2S8/c1py12PSVWPwuHT8lFUv1Ywd+M+np797VrpPaM4vf5WMd734SjI+PDCQjLdScPSN2P9oROxtwveYWZs4alpiZtZ5IpRVS6zRmgbwA0kbJK1sRoXMrLUqHftdpZZ6JPVK+pGkzZI2SfpcUX6CpIclvVz8nFeUS9LXJG2V9IKkM+sdo9Ekdm5EnAl8Erha0nnj/CFWSlovaf1hDjZ4ODObepU59sssJQwCn4+IpcA5VPLEUuA6YF1ELAHWFZ+hkkuWFMtK4NZ6B2goiUXEzuLnHuB+4KxxtlkVEX0R0dfNzEYOZ2bToNKxr1JL3e+K6I+IZ4v1AWALsBBYDqwpNlsDXFysLwfujIofA8dL6kkdY9JJTNJsSXOPrAMfBzZO9vvMrH0McUypZSIkLQbOAJ4GFkREfxHaBRy5Rb8QeLVqtx1FWU2NdOwvAO5XZfjADOBbEfH9Br7PzNrABEfsz5e0vurzqohYNXojSXOAe4FrI2K/qoYdRURImvScTpNOYhHxCvDhye7fSY6ZNSsZ/9Uf/GYyvuPKt5Pxgwe7k/G5s9+pGVty4mvJfd8z80Ay/vWFTyfjQzGcjD9xsPb/ret1DL96+MRk/CcD70vG/6mrdp/wH376n5L7Xjf/+WT8qXfSXSP/866rkvFT/vapZLzVJvCikL31xodK6qaSwO6KiPuK4t2SeiKiv7hcPDLgcCfQW7X7oqKspnzuo5rZtIiAw8PHlFrqUaXJdTuwJSJuqQqtBVYU6yuAB6rKP1PcpTwHeLPqsnNcHidmZiNULieb1r75CHAF8KKk54qy64GbgHskXQlsBy4tYg8CFwFbgbeBP613ACcxMxujWSP2I+JxqPllF4yzfQBXT+QYTmJmNsKRIRa5cBIzs1HyeuzISczMxvAc+zbCWwvSp/muM1cn48tmtu5Jh3pDKLqU/j/2ecnRJ0PJfdfxejL+xRf/WzJO7+GaoS0H3pvc9Y/3n5SM/3RbbzJ+6g/zfcSucnfSr2wzs0x5emozy54vJ80sW747aWbZ891JM8tWhBh0EjOznPly0syy5T6xo9DwwfSYoONfqT1VDsDl6z+bjH/s1NqvHgM4c872mrE/eXf6lWoHo/ZYKoDH30lPM3Tvvt9Jxk8/rvbxrzx+U3LfL237dDL+/m+kX5s2PLP2X+/Xu05J7lvvVXYfeCM9hm1w2y+T8XbnJGZm2fI4MTPLnseJmVm2ImCwxISH7cJJzMzG8OWkmWXLfWJmlr1wEjOznLlj/2gT6VfmdT//SjJ+0q2nJ+PPLPjtZPx759Z+NdnvfeqWmjGAlw79ejL+V3evSMZP+7v0WK+fLz23ZuzvP/3x5L7H7kt3LvdueDIZT/0zrPdPtN5LEAfrxHMW4T4xM8uaGPLdSTPLmfvEzCxbfnbSzPIWdbt524qTmJmN4buTZpatcMe+meWuoy4nJa0GPgXsiYgPFWUnAN8BFgPbgEsj4o2pq2behv71zWS8+7Hnk/Ffq/P9h2fXntPr/vOWJff9x/4PJuOnfD89Z9fQ/v3JuNZvrhl7/670ux05Jt0a6OSxWq2W093JMm3GO4ALR5VdB6yLiCXAuuKzmXWAiEoSK7O0g7pJLCIeA/aNKl4OrCnW1wAXN7leZtZCw6FSSzuYbJ/YgojoL9Z3AQuaVB8zawMd1SdWT0SEpJp/ZEkrgZUAs3hXo4czsykWiOGM7k5Otqa7JfUAFD9rvg0iIlZFRF9E9HUzc5KHM7PpFCWXdjDZJLYWODK9wQrggeZUx8xartM69iV9G3gKeL+kHZKuBG4CPibpZeC/FJ/NrFM0qSkmabWkPZI2VpXdIGmnpOeK5aKq2N9I2irpZ5I+UaaqdfvEIuLyGqELyhzA6ovBxkY8zd41VDP20lvvTe57Re+Pk/HbFqdvPP/aE8lw8s+W+7sZO1kTW1l3AF8H7hxV/pWI+HJ1gaSlwGXAB4GTgEck/UZE1P4LzuQvJ82sQwUwPKxSS93vGn+IVi3Lgbsj4mBE/ALYCpxVbycnMTMbKYBQuWXyrpH0QnG5Oa8oWwi8WrXNjqIsyUnMzMaIKLcA8yWtr1pWlvj6W4HTgGVAP3BzI3X1A+BmNlb58RN7I6JvQl8dsfvIuqTbgO8VH3cCvVWbLirKktwSM7NRyg2vmGzn/5ExpoVLgCN3LtcCl0maKelUYAnwk3rf55aYmY3VpJGsxRCt86lcdu4AvgCcL2lZcZRtwFUAEbFJ0j3AZiqTlFxd784kOIl1hOMerj2Vz5Nn136dG8Bn//jRZHzvGeljn7gu/djs4K7dybi1oYAoceex1FeNP0Tr9sT2NwI3TuQYTmJmNo72GI1fhpOYmY3VLg9GluAkZmZjOYmZWbaODHbNhJOYmY1xVE2KaGYdqEl3J6eDk5iZjVF7rub24yTWAeLgwZqx07+5I7nvrRf8QTL+vy+5LRn/yyWfTsYHHz+tZqz3//wsue/Q3teTcZsi7TRtawlOYmY2SsMzVEwrJzEzG8stMTPL2nCrK1Cek5iZjeRxYmaWO9+dNLO8ZZTEPCmimWXNLbEON7j91WR8281nJ+MDN61Pxv/5t+9Ixjf/VlfN2GeHP5fct/c725PxwZ2vJeNZPTvTZnw5aWb5CvzYkZllzi0xM8uZLyfNLG9OYmaWNScxM8uVwpeTZpa7Tro7KWk18ClgT0R8qCi7Afgz4F+Kza6PiAenqpI2dWbfnx4H9vV9lybj/++m9Jxgt/U+UTN2zWe/m9z3a90XJ+Mnf/OdZNzzkU1eTi2xMiP27wAuHKf8KxGxrFicwMw6SZRc2kDdllhEPCZp8dRXxczaQmZ9Yo08O3mNpBckrZY0r2k1MrPWy6glNtkkditwGrAM6AdurrWhpJWS1ktaf5jac8GbWfvQcLmlHUwqiUXE7ogYiohh4DbgrMS2qyKiLyL6upk52XqamY1rUklMUk/Vx0uAjc2pjpm1hYwuJ8sMsfg2cD4wX9IO4AvA+ZKWUfljbAOumsI6mtl0yqxjv8zdycvHKb59CupirTA8lAx3Pfp8Mv7qn38gGT/1qqU1Y5su+kZy352XP5qM33/495PxnpufTMYtoZOSmJkdhZzEzCxXon3uPJbhOfbNbKT4j4fA6y31FONI90jaWFV2gqSHJb1c/JxXlEvS1yRtLcagnlmmuk5iZjZW8+5O3sHYxxavA9ZFxBJgXfEZ4JPAkmJZSWU8al1OYmY2VpOSWEQ8BuwbVbwcWFOsrwEuriq/Myp+DBw/ajjXuJzEzGyMZl1O1rAgIvqL9V3AgmJ9IVD9eq4dRVmSO/aPdmps3qj46UvJ+IJHf6dmrPu/1n6dG8BfnJieJujJi96XjM+4Z1HN2OCrO5L7HvXKJ6j5kqp/UasiYlXpw0SE1NioNCcxMxspJnR3cm9E9E3wCLsl9UREf3G5uKco3wn0Vm23qChL8uWkmY01tY8drQVWFOsrgAeqyj9T3KU8B3iz6rKzJrfEzGyMZj12VOOxxZuAeyRdCWwHjkwf/CBwEbAVeBv40zLHcBIzs7GalMRqPLYIcME42wZw9USP4SRmZiO10QwVZTiJmdkIosNmsTCzo4+TmE0rzaw9Y27X/BOT+w71nJCM7/3w3GT89bMHk/Hf/WDtV7p1Kz1O7O2h9DRBuwfmJOO93b9Kxi3BSczMsuYkZmbZ6rSZXc3sKOQkZmY5y2lSRCcxMxvDl5Nmli8PdjWz7DmJ2UQcM2tWMq5Te5PxNz9Ue6zX7nPS84VdeN5Pk/FVC9Yl43OVngjlmOR8Zccl933+UHqM2+Fn5yXjg69sScZtfB6xb2bZ03A+WcxJzMxGcp+YmeXOl5NmljcnMTPLmVtiZpY3JzEzy9bE3nbUcnWTmKRe4E4qL7gMKu+V+6qkE4DvAIuBbcClEfHG1FW1fXW9+93pDRa9NxnuP39+Mv7m2e8k43/04Sdrxq4+4ankvj0z0nNyQb142hPv1P7XcO3mS2vGAN5+In1eTl43MKk6WVpu48TKvLJtEPh8RCwFzgGulrQUuA5YFxFLgHXFZzPrBBHlljZQN4lFRH9EPFusDwBbqLxafDmwpthsDXDxVFXSzKaXotzSDibUJyZpMXAG8DSwoOrFlruoXG6aWe46dbCrpDnAvcC1EbFfVc/ERURI4+dlSSuBlQCzeFdjtTWzaZFTx36ZPjEkdVNJYHdFxH1F8W5JPUW8B9gz3r4RsSoi+iKir5vaL7Qws/ah4XJLO6ibxFRpct0ObImIW6pCa4EVxfoK4IHmV8/Mpl2QVcd+mcvJjwBXAC9Keq4oux64CbhH0pXAdiB9v3yKpV5bBqDklDCg49LTwmz78/9UMzb0WweS+/bM25+MX7Uonf//cO7Lyfi8rtqX6XuH0n/uLYfeTsYffXtJMv6lH34qGV/4w9qx+T9Pnxf9YmMyPrQ/vb9NXrt02pdRN4lFxONUho6M54LmVsfM2kInJTEzO7rkNtjVSczMRorwpIhmlrl8cpiTmJmN5ctJM8tXAL6cNLOs5ZPDOieJHTz/N5Px1/7HoWT8sg9sSMb/du4jNWOLZvwque+CrvQYtG51JeMHozsZ/+5btafL+buf//fkvgf+MT1N0KL/+1oy/oF9LyXjw2/VPjfDh9O/E2udZl5OStoGDABDwGBE9DVzKq9Sjx2Z2dFFw1FqmYCPRsSyiOgrPjdtKi8nMTMbKSawTF7TpvJyEjOzESqDXaPUUlIAP5C0oZjVBpo4lVfH9ImZWROVn6FivqT1VZ9XRcSqUducGxE7Jb0HeFjSiI7U1FReZTiJmdkYE2hl7a3q5xpXROwsfu6RdD9wFsVUXhHRn5rKqwxfTprZSE3sE5M0W9LcI+vAx4GNNHEqL7fEzGyUpj47uQC4v5gKawbwrYj4vqRnaNJUXh2TxHadc2wy/t2zv5aMn96dno8sNZbrB2/PS+57bf/vJ+Nzuw8m409uPzUZP3ZD7XFiJ9+1LbnvnNfSr3QbbJOJ72yaNen3HhGvAB8ep/x1mjSVV8ckMTNrkk57ea6ZHYUyaoE7iZnZWPnkMCcxMxtLw/lcTzqJmdlIwUQGu7ack5iZjSAm9EhRyzmJmdlYTmLT7+QvPpmMX/vF30vGZyw8KRk/vPg9NWOHjk/P99X1Trptvvvd6V/D+x7ZnIwPDwzUjA0m9zSrwUnMzLLlPjEzy53vTppZxsKXk2aWscBJzMwyl8/VpJOYmY3lcWJmlrdOSmKSeoE7qUxuFlTm0P6qpBuAPwP+pdj0+oh4cKoqOtUGd6bfr6hEPD0TWX31fgkZteytE0TAUD5/68q0xAaBz0fEs8U0sxskPVzEvhIRX5666plZS3RSS6x4rVJ/sT4gaQuwcKorZmYtlFESm9CLQiQtBs4Ani6KrpH0gqTVksado1nSSknrJa0/THoaZjNrAwEMR7mlDZROYpLmAPcC10bEfuBW4DRgGZWW2s3j7RcRqyKiLyL6uhvuPTKzqRcQw+WWNlDq7qSkbioJ7K6IuA8gInZXxW8DvjclNTSz6RVk1bFftyWmyruWbge2RMQtVeU9VZtdQuVdcmbWCSLKLW2gTEvsI8AVwIuSnivKrgcul7SMSt7eBlw1JTU0s+nXJgmqjDJ3Jx8HNE4o2zFhZpbSPq2sMjxi38xGCsBT8ZhZ1twSM7N8dd5jR2Z2NAmINhkDVoaTmJmN1Saj8ctwEjOzsdwnZmbZivDdSTPLnFtiZpavIIaGWl2J0pzEzGykI1PxZGJC84mZ2VGiiVPxSLpQ0s8kbZV0XbOr6paYmY0QQDSpJSapC/gG8DFgB/CMpLURsbkpB8AtMTMbLZo6KeJZwNaIeCUiDgF3A8ubWV23xMxsjCZ27C8EXq36vAM4u1lfDtOcxAZ4Y+8j8Q/bq4rmA3unsw4T0K51a9d6ges2Wc2s2ymNfsEAbzz0SPzD/JKbz5K0vurzqohY1WgdJmJak1hE/Hr1Z0nrI6JvOutQVrvWrV3rBa7bZLVb3SLiwiZ+3U6gt+rzoqKsadwnZmZT6RlgiaRTJR0LXAasbeYB3CdmZlMmIgYlXQM8BHQBqyNiUzOP0eokNq3XzhPUrnVr13qB6zZZ7Vy3hkXEg0zhdPaKjJ6RMjMbzX1iZpa1liSxqX4MoRGStkl6UdJzo24dt6IuqyXtkbSxquwESQ9Lern4Oa+N6naDpJ3FuXtO0kUtqluvpB9J2ixpk6TPFeUtPXeJerXFecvVtF9OFo8h/H+qHkMALm/mYwiNkLQN6IuIlo8pknQecAC4MyI+VJR9CdgXETcV/wOYFxF/3SZ1uwE4EBFfnu76jKpbD9ATEc9KmgtsAC4G/oQWnrtEvS6lDc5brlrREpvyxxA6RUQ8BuwbVbwcWFOsr6Hyj2Da1ahbW4iI/oh4tlgfALZQGTne0nOXqJc1oBVJbLzHENrpFxnADyRtkLSy1ZUZx4KI6C/WdwELWlmZcVwj6YXicrMll7rVJC0GzgCepo3O3ah6QZudt5y4Y3+scyPiTOCTwNXFZVNbikpfQDvdXr4VOA1YBvQDN7eyMpLmAPcC10bE/upYK8/dOPVqq/OWm1YksSl/DKEREbGz+LkHuJ/K5W872V30rRzpY9nT4vr8u4jYHRFDUXnf12208NxJ6qaSKO6KiPuK4pafu/Hq1U7nLUetSGJT/hjCZEmaXXS4Imk28HFgY3qvabcWWFGsrwAeaGFdRjiSIAqX0KJzJ0nA7cCWiLilKtTSc1erXu1y3nLVksGuxS3k/8V/PIZw47RXYhyS3kel9QWVpxm+1cq6Sfo2cD6VWQ52A18AvgvcA5wMbAcujYhp72CvUbfzqVwSBbANuKqqD2o663Yu8M/Ai8CRSa+up9L/1LJzl6jX5bTBecuVR+ybWdbcsW9mWXMSM7OsOYmZWdacxMwsa05iZpY1JzEzy5qTmJllzUnMzLL2b3H4c4GVGcTSAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0W3D61sza61Z",
        "outputId": "fbf5b723-fd93-492e-a8e7-4bc1ff33509b"
      },
      "source": [
        "#Augmented training sets: x_train_aug, y_train_aug\n",
        "#vectorize data into 2-d np array\n",
        "x_train, x_test = x_train.reshape(x_train.shape[0], x_train.shape[1]*x_train.shape[2]), x_test.reshape(x_test.shape[0], x_test.shape[1] * x_test.shape[2])\n",
        "\n",
        "#normalization: rescale the images from [0,255] to the [0.0,1.0] range.\n",
        "x_train, x_test = x_train/255.0, x_test/255.0\n",
        "print(\"Dimansion of x_train: \", x_train.shape)\n",
        "\n",
        "#reshape and normalize the augmented training set\n",
        "#x_train_aug = x_train_aug.reshape(x_train_aug.shape[0], x_train_aug.shape[1]*x_train_aug.shape[2])\n",
        "#x_train_aug = x_train_aug/255.0\n",
        "#print(\"Dimansion of x_train_aug: \", x_train_aug.shape)\n",
        "\n",
        "#take the first 100 as validation set\n",
        "x_validation = x_train[:100]\n",
        "y_validation = y_train[:100]\n",
        "x_train = x_train[101:]\n",
        "y_train = y_train[101:]\n",
        "\n",
        "#unormalized data\n",
        "#x_train_unormalized = x_train * 255.0\n",
        "#x_test_unormalized = x_test * 255.0\n",
        "#x_validation_unormalized = x_validation * 255.0\n",
        "\n",
        "#At this point x_train, x_test, x_train_unormalized, x_validation_unormalized, x_test_unormalized, x_validation are 2-d np array, y_train, y_test, y_validation are 1-d np array"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Dimansion of x_train:  (60000, 784)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BvYDSGCr3eGa"
      },
      "source": [
        "x_validation_aug = x_train_aug[:100]\n",
        "y_validation_aug = y_train_aug[:100]\n",
        "x_train_aug = x_train_aug[101:]\n",
        "y_train_aug = y_train_aug[101:]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Kj9CrRTlrged"
      },
      "source": [
        "# preprocess data, normalization by substracting training mean from training, validation, test datasets\n",
        "def prepro(X_train, X_val, X_test):\n",
        "    mean = np.mean(X_train)\n",
        "    return X_train - mean, X_val - mean, X_test - mean"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LIV9nDb1CNJo"
      },
      "source": [
        "Hyper-param tuning"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Xexn-q0OCMWh"
      },
      "source": [
        "def tune_hyperparam(hidden_units, model, n_iter, alpha, mb_size, n_experiment, reg, print_after, p_dropout, loss, nonlin, solver_fun, max_norm, x_train, x_val, x_test, y_train, y_val, y_test, candidates, nlayer):\n",
        "    M, D, C = x_train.shape[0], x_train.shape[1], y_train.max() + 1\n",
        "\n",
        "    accuracies = np.array([[0,0,0,0]]) # initiate accuracy array, containing hyper-param candidates composition and corresponding accuracy\n",
        "    accs = np.zeros(n_experiment) # initiate accuracy array for a single set of hyper-param\n",
        "\n",
        "    for max_norm in candidates[0]:\n",
        "      for dropout_rate_input in candidates[1]:\n",
        "        for dropout_rate_hidden in candidates[2]:\n",
        "          # composite the droup out array based on the number of hidden layers\n",
        "          dropoutArray = np.array([dropout_rate_input])\n",
        "          for i in range(1):\n",
        "            dropoutArray = np.append(dropoutArray, [dropout_rate_hidden])\n",
        "\n",
        "          print('Experiment on hyper-parm candidates. max_norm: ', max_norm, 'input drop out rate: ', dropout_rate_input, 'hidden layer drop out rate: ', dropout_rate_hidden)\n",
        "\n",
        "          for k in range(n_experiment):\n",
        "             print('Experiment-{}'.format(k + 1))\n",
        "\n",
        "             # Reset model\n",
        "             if model == 'ff':\n",
        "                net = nn.FeedForwardNet(D, C, H=hidden_units, lam=reg, p_dropout=dropoutArray, loss=loss, nonlin=nonlin, nlayer=nlayer)\n",
        "             elif model == 'cnn':\n",
        "                net = nn.ConvNet(10, C, H=128, lam=1e-3, p_dropout=dropoutArray, loss=loss, nonlin=nonlin)\n",
        "\n",
        "             net = solver_fun(\n",
        "                 net, x_train, y_train, val_set=(x_val, y_val), mb_size=mb_size, alpha=alpha,\n",
        "                 n_iter=n_iter, print_after=print_after, max_norm=max_norm\n",
        "             )\n",
        "\n",
        "             y_pred = net.predict(x_test)\n",
        "             accs[k] = np.mean(y_pred == y_test)\n",
        "\n",
        "          print()\n",
        "          print('Mean accuracy: {:.4f}, std: {:.4f}'.format(accs.mean(), accs.std()))\n",
        "          print()\n",
        "          \n",
        "          accuracies = np.concatenate((accuracies, [[max_norm, dropout_rate_input, dropout_rate_hidden, accs.mean()]]))\n",
        "\n",
        "      \n",
        "    \n",
        "    index = np.argmax(accuracies, axis=0)[3] # index of the best hyper-parm in the accuracies\n",
        "    best_max_norm = accuracies[index][0]\n",
        "    best_dropout_rate_input = accuracies[index][1]\n",
        "    best_dropout_rate_hidden = accuracies[index][2]\n",
        "    best_acc = accuracies[index][3]\n",
        "\n",
        "         \n",
        "    return best_max_norm, best_dropout_rate_input, best_dropout_rate_hidden, best_acc\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HudecacFZKBo"
      },
      "source": [
        "# Train and predict mnist dataset\n",
        "def run_mnist(hidden_units, model, n_iter, alpha, mb_size, n_experiment, reg, print_after, p_dropout, loss, nonlin, solver, max_norm, x_train, x_val, x_test, y_train, y_val, y_test, candidates, nlayer):\n",
        "    M, D, C = x_train.shape[0], x_train.shape[1], y_train.max() + 1\n",
        "  \n",
        "    x_train, x_val, x_test = prepro(x_train, x_val, x_test)\n",
        "    \n",
        "    # A key/value hash table of optimization algorithm, all algorithms use minibatch technique\n",
        "    solvers = dict(\n",
        "        sgd=sgd,                # SGD\n",
        "        momentum=momentum,          # Momentum SGD\n",
        "        nesterov=nesterov,          # Nesterov Momentum\n",
        "        adagrad=adagrad,           # Adagrad\n",
        "        rmsprop=rmsprop,          # RMSprop\n",
        "        adam=adam             # Adam\n",
        "    )\n",
        "    \n",
        "    solver_fun = solvers[solver]\n",
        "    \n",
        "    print()\n",
        "    print('Experimenting on {}'.format(solver))\n",
        "    print()\n",
        "    \n",
        "    if model == 'cnn':\n",
        "        img_shape = (1, 28, 28)\n",
        "        x_train = x_train.reshape(-1, *img_shape)\n",
        "        x_val = x_val.reshape(-1, *img_shape)\n",
        "        x_test = x_test.reshape(-1, *img_shape)\n",
        "\n",
        "    #tune hyper-parameters\n",
        "    best_max_norm, best_dropout_rate_input, best_dropout_rate_hidden, best_acc = tune_hyperparam(hidden_units, model, 500, alpha, mb_size, n_experiment, reg, print_after, p_dropout, loss, nonlin, solver_fun, max_norm, x_train, x_val, x_test, y_train, y_val, y_test, candidates, nlayer)\n",
        "\n",
        "    #train and predict\n",
        "    accs = np.zeros(n_experiment) # initiate accuracy array for a single set of hyper-param\n",
        "    best_dropoutArray = np.array([best_dropout_rate_input])\n",
        "    for i in range(1):\n",
        "        best_dropoutArray = np.append(best_dropoutArray, [best_dropout_rate_hidden])\n",
        "\n",
        "    for k in range(n_experiment):\n",
        "        print('Experiment-{}'.format(k + 1))\n",
        "\n",
        "        # Reset model\n",
        "        if model == 'ff':\n",
        "           net = nn.FeedForwardNet(D, C, H=hidden_units, lam=reg, p_dropout=best_dropoutArray, loss=loss, nonlin=nonlin, nlayer=nlayer)\n",
        "        elif model == 'cnn':\n",
        "           net = nn.ConvNet(10, C, H=128, lam=1e-3, p_dropout=best_dropoutArray, loss=loss, nonlin=nonlin)\n",
        "\n",
        "        net = solver_fun(\n",
        "            net, x_train, y_train, val_set=(x_val, y_val), mb_size=mb_size, alpha=alpha,\n",
        "            n_iter=n_iter, print_after=print_after, max_norm=best_max_norm\n",
        "        )\n",
        "\n",
        "        y_pred = net.predict(x_test)\n",
        "        accs[k] = np.mean(y_pred == y_test)\n",
        "\n",
        "    print()\n",
        "    print('Best max_norm: ', best_max_norm, 'best dropout rate for input: ', best_dropout_rate_input, 'best dropout rate for hidden layers: ', best_dropout_rate_hidden, 'and result accuracy is: ', accs.mean())\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Lysvds1wK2mE"
      },
      "source": [
        "Dropout NN + max-norm constraint + ReLU + 3 layers + 1024 hidden units (100,000 weight updates)\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PmQUBQJ_segS",
        "outputId": "8a1b5bfd-8305-47d3-b0f5-2f97f518845b"
      },
      "source": [
        "model = 'ff'\n",
        "n_iter = 100000\n",
        "alpha = 1e-3\n",
        "mb_size = 64\n",
        "n_experiment = 1\n",
        "reg = 1e-5\n",
        "print_after = 100\n",
        "p_dropout = [0.8,0.5] #if nlayer=n, len(p_dropout) = n-1\n",
        "loss = 'cross_ent'\n",
        "nonlin = 'relu'\n",
        "solver = 'sgd'\n",
        "max_norm = 4\n",
        "nlayer = 3\n",
        "\n",
        "# Define default candidates\n",
        "candidates = np.array([[3, 3.5, 4],[0.795, 0.8, 0.85],[0.495, 0.5, 0.55]]) # first hyper-param as max-norm, second as drop-out rate for input layer, third as drop-out rate for hidden layers\n",
        "\n",
        "run_mnist(1024, model, n_iter, alpha, mb_size, n_experiment, reg, print_after, p_dropout, loss, nonlin, solver, max_norm, x_train, x_validation, x_test, y_train, y_validation, y_test, candidates, nlayer)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Experimenting on sgd\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  3.0 input drop out rate:  0.795 hidden layer drop out rate:  0.495\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.9418 validation: 0.800000\n",
            "Iter-200 loss: 1.6889 validation: 0.820000\n",
            "Iter-300 loss: 1.6383 validation: 0.820000\n",
            "Iter-400 loss: 1.4346 validation: 0.850000\n",
            "Iter-500 loss: 1.4446 validation: 0.860000\n",
            "\n",
            "Mean accuracy: 0.8596, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  3.0 input drop out rate:  0.795 hidden layer drop out rate:  0.5\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.9109 validation: 0.850000\n",
            "Iter-200 loss: 1.7611 validation: 0.860000\n",
            "Iter-300 loss: 1.5916 validation: 0.860000\n",
            "Iter-400 loss: 1.4576 validation: 0.860000\n",
            "Iter-500 loss: 1.4488 validation: 0.880000\n",
            "\n",
            "Mean accuracy: 0.8616, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  3.0 input drop out rate:  0.795 hidden layer drop out rate:  0.55\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.9078 validation: 0.790000\n",
            "Iter-200 loss: 1.6824 validation: 0.820000\n",
            "Iter-300 loss: 1.5460 validation: 0.820000\n",
            "Iter-400 loss: 1.5342 validation: 0.850000\n",
            "Iter-500 loss: 1.5115 validation: 0.880000\n",
            "\n",
            "Mean accuracy: 0.8619, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  3.0 input drop out rate:  0.8 hidden layer drop out rate:  0.495\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.9127 validation: 0.850000\n",
            "Iter-200 loss: 1.7082 validation: 0.850000\n",
            "Iter-300 loss: 1.5493 validation: 0.850000\n",
            "Iter-400 loss: 1.5245 validation: 0.870000\n",
            "Iter-500 loss: 1.4088 validation: 0.880000\n",
            "\n",
            "Mean accuracy: 0.8605, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  3.0 input drop out rate:  0.8 hidden layer drop out rate:  0.5\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.8807 validation: 0.810000\n",
            "Iter-200 loss: 1.7027 validation: 0.830000\n",
            "Iter-300 loss: 1.6165 validation: 0.860000\n",
            "Iter-400 loss: 1.5187 validation: 0.870000\n",
            "Iter-500 loss: 1.3814 validation: 0.870000\n",
            "\n",
            "Mean accuracy: 0.8559, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  3.0 input drop out rate:  0.8 hidden layer drop out rate:  0.55\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.8851 validation: 0.810000\n",
            "Iter-200 loss: 1.6557 validation: 0.840000\n",
            "Iter-300 loss: 1.5987 validation: 0.850000\n",
            "Iter-400 loss: 1.4854 validation: 0.890000\n",
            "Iter-500 loss: 1.4852 validation: 0.900000\n",
            "\n",
            "Mean accuracy: 0.8587, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  3.0 input drop out rate:  0.85 hidden layer drop out rate:  0.495\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.9157 validation: 0.830000\n",
            "Iter-200 loss: 1.6755 validation: 0.860000\n",
            "Iter-300 loss: 1.5482 validation: 0.870000\n",
            "Iter-400 loss: 1.3996 validation: 0.880000\n",
            "Iter-500 loss: 1.4660 validation: 0.890000\n",
            "\n",
            "Mean accuracy: 0.8637, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  3.0 input drop out rate:  0.85 hidden layer drop out rate:  0.5\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.8302 validation: 0.820000\n",
            "Iter-200 loss: 1.6973 validation: 0.840000\n",
            "Iter-300 loss: 1.5709 validation: 0.840000\n",
            "Iter-400 loss: 1.4809 validation: 0.870000\n",
            "Iter-500 loss: 1.3574 validation: 0.880000\n",
            "\n",
            "Mean accuracy: 0.8642, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  3.0 input drop out rate:  0.85 hidden layer drop out rate:  0.55\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.8882 validation: 0.810000\n",
            "Iter-200 loss: 1.6781 validation: 0.840000\n",
            "Iter-300 loss: 1.5425 validation: 0.840000\n",
            "Iter-400 loss: 1.5077 validation: 0.850000\n",
            "Iter-500 loss: 1.4407 validation: 0.860000\n",
            "\n",
            "Mean accuracy: 0.8572, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  3.5 input drop out rate:  0.795 hidden layer drop out rate:  0.495\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.7664 validation: 0.840000\n",
            "Iter-200 loss: 1.5346 validation: 0.860000\n",
            "Iter-300 loss: 1.3878 validation: 0.850000\n",
            "Iter-400 loss: 1.2986 validation: 0.860000\n",
            "Iter-500 loss: 1.3494 validation: 0.890000\n",
            "\n",
            "Mean accuracy: 0.8674, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  3.5 input drop out rate:  0.795 hidden layer drop out rate:  0.5\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.7636 validation: 0.790000\n",
            "Iter-200 loss: 1.5765 validation: 0.850000\n",
            "Iter-300 loss: 1.3941 validation: 0.850000\n",
            "Iter-400 loss: 1.3108 validation: 0.880000\n",
            "Iter-500 loss: 1.2011 validation: 0.900000\n",
            "\n",
            "Mean accuracy: 0.8733, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  3.5 input drop out rate:  0.795 hidden layer drop out rate:  0.55\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.8350 validation: 0.790000\n",
            "Iter-200 loss: 1.5087 validation: 0.830000\n",
            "Iter-300 loss: 1.3333 validation: 0.850000\n",
            "Iter-400 loss: 1.3176 validation: 0.890000\n",
            "Iter-500 loss: 1.2964 validation: 0.890000\n",
            "\n",
            "Mean accuracy: 0.8657, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  3.5 input drop out rate:  0.8 hidden layer drop out rate:  0.495\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.8091 validation: 0.800000\n",
            "Iter-200 loss: 1.5374 validation: 0.840000\n",
            "Iter-300 loss: 1.3685 validation: 0.860000\n",
            "Iter-400 loss: 1.3533 validation: 0.890000\n",
            "Iter-500 loss: 1.2444 validation: 0.880000\n",
            "\n",
            "Mean accuracy: 0.8665, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  3.5 input drop out rate:  0.8 hidden layer drop out rate:  0.5\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.7569 validation: 0.810000\n",
            "Iter-200 loss: 1.5975 validation: 0.840000\n",
            "Iter-300 loss: 1.3671 validation: 0.870000\n",
            "Iter-400 loss: 1.3107 validation: 0.880000\n",
            "Iter-500 loss: 1.2452 validation: 0.880000\n",
            "\n",
            "Mean accuracy: 0.8714, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  3.5 input drop out rate:  0.8 hidden layer drop out rate:  0.55\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.7596 validation: 0.790000\n",
            "Iter-200 loss: 1.5189 validation: 0.810000\n",
            "Iter-300 loss: 1.3717 validation: 0.850000\n",
            "Iter-400 loss: 1.3182 validation: 0.870000\n",
            "Iter-500 loss: 1.2521 validation: 0.890000\n",
            "\n",
            "Mean accuracy: 0.8696, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  3.5 input drop out rate:  0.85 hidden layer drop out rate:  0.495\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.7807 validation: 0.810000\n",
            "Iter-200 loss: 1.5365 validation: 0.830000\n",
            "Iter-300 loss: 1.3737 validation: 0.860000\n",
            "Iter-400 loss: 1.2665 validation: 0.880000\n",
            "Iter-500 loss: 1.2472 validation: 0.900000\n",
            "\n",
            "Mean accuracy: 0.8705, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  3.5 input drop out rate:  0.85 hidden layer drop out rate:  0.5\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.8128 validation: 0.830000\n",
            "Iter-200 loss: 1.4799 validation: 0.850000\n",
            "Iter-300 loss: 1.3851 validation: 0.890000\n",
            "Iter-400 loss: 1.3569 validation: 0.890000\n",
            "Iter-500 loss: 1.2734 validation: 0.900000\n",
            "\n",
            "Mean accuracy: 0.8749, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  3.5 input drop out rate:  0.85 hidden layer drop out rate:  0.55\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.7735 validation: 0.820000\n",
            "Iter-200 loss: 1.5132 validation: 0.860000\n",
            "Iter-300 loss: 1.3785 validation: 0.840000\n",
            "Iter-400 loss: 1.2930 validation: 0.880000\n",
            "Iter-500 loss: 1.1709 validation: 0.900000\n",
            "\n",
            "Mean accuracy: 0.8778, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  4.0 input drop out rate:  0.795 hidden layer drop out rate:  0.495\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.7020 validation: 0.830000\n",
            "Iter-200 loss: 1.3588 validation: 0.820000\n",
            "Iter-300 loss: 1.1295 validation: 0.850000\n",
            "Iter-400 loss: 1.1438 validation: 0.900000\n",
            "Iter-500 loss: 1.0967 validation: 0.910000\n",
            "\n",
            "Mean accuracy: 0.8767, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  4.0 input drop out rate:  0.795 hidden layer drop out rate:  0.5\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.6776 validation: 0.820000\n",
            "Iter-200 loss: 1.3671 validation: 0.860000\n",
            "Iter-300 loss: 1.2217 validation: 0.870000\n",
            "Iter-400 loss: 1.1507 validation: 0.880000\n",
            "Iter-500 loss: 1.1136 validation: 0.900000\n",
            "\n",
            "Mean accuracy: 0.8721, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  4.0 input drop out rate:  0.795 hidden layer drop out rate:  0.55\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.6499 validation: 0.820000\n",
            "Iter-200 loss: 1.3562 validation: 0.850000\n",
            "Iter-300 loss: 1.2650 validation: 0.880000\n",
            "Iter-400 loss: 1.1991 validation: 0.880000\n",
            "Iter-500 loss: 1.1295 validation: 0.880000\n",
            "\n",
            "Mean accuracy: 0.8724, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  4.0 input drop out rate:  0.8 hidden layer drop out rate:  0.495\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.6869 validation: 0.810000\n",
            "Iter-200 loss: 1.3614 validation: 0.860000\n",
            "Iter-300 loss: 1.3235 validation: 0.870000\n",
            "Iter-400 loss: 1.1610 validation: 0.870000\n",
            "Iter-500 loss: 1.0057 validation: 0.900000\n",
            "\n",
            "Mean accuracy: 0.8746, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  4.0 input drop out rate:  0.8 hidden layer drop out rate:  0.5\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.7245 validation: 0.820000\n",
            "Iter-200 loss: 1.3939 validation: 0.850000\n",
            "Iter-300 loss: 1.2675 validation: 0.880000\n",
            "Iter-400 loss: 1.0768 validation: 0.900000\n",
            "Iter-500 loss: 1.0864 validation: 0.910000\n",
            "\n",
            "Mean accuracy: 0.8790, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  4.0 input drop out rate:  0.8 hidden layer drop out rate:  0.55\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.7089 validation: 0.850000\n",
            "Iter-200 loss: 1.4251 validation: 0.860000\n",
            "Iter-300 loss: 1.2075 validation: 0.860000\n",
            "Iter-400 loss: 1.1226 validation: 0.880000\n",
            "Iter-500 loss: 1.0866 validation: 0.890000\n",
            "\n",
            "Mean accuracy: 0.8793, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  4.0 input drop out rate:  0.85 hidden layer drop out rate:  0.495\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.6267 validation: 0.820000\n",
            "Iter-200 loss: 1.3140 validation: 0.860000\n",
            "Iter-300 loss: 1.0842 validation: 0.890000\n",
            "Iter-400 loss: 1.1303 validation: 0.890000\n",
            "Iter-500 loss: 1.1270 validation: 0.910000\n",
            "\n",
            "Mean accuracy: 0.8826, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  4.0 input drop out rate:  0.85 hidden layer drop out rate:  0.5\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.5914 validation: 0.850000\n",
            "Iter-200 loss: 1.3413 validation: 0.880000\n",
            "Iter-300 loss: 1.3622 validation: 0.890000\n",
            "Iter-400 loss: 1.1032 validation: 0.910000\n",
            "Iter-500 loss: 0.9983 validation: 0.920000\n",
            "\n",
            "Mean accuracy: 0.8852, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  4.0 input drop out rate:  0.85 hidden layer drop out rate:  0.55\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.7157 validation: 0.830000\n",
            "Iter-200 loss: 1.5110 validation: 0.840000\n",
            "Iter-300 loss: 1.1746 validation: 0.880000\n",
            "Iter-400 loss: 1.1120 validation: 0.890000\n",
            "Iter-500 loss: 1.0423 validation: 0.890000\n",
            "\n",
            "Mean accuracy: 0.8736, std: 0.0000\n",
            "\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.5996 validation: 0.830000\n",
            "Iter-200 loss: 1.3933 validation: 0.860000\n",
            "Iter-300 loss: 1.2065 validation: 0.850000\n",
            "Iter-400 loss: 1.0745 validation: 0.910000\n",
            "Iter-500 loss: 0.9967 validation: 0.920000\n",
            "Iter-600 loss: 0.9587 validation: 0.910000\n",
            "Iter-700 loss: 0.8759 validation: 0.920000\n",
            "Iter-800 loss: 0.9923 validation: 0.940000\n",
            "Iter-900 loss: 0.8901 validation: 0.940000\n",
            "Iter-1000 loss: 0.8192 validation: 0.950000\n",
            "Iter-1100 loss: 0.8368 validation: 0.950000\n",
            "Iter-1200 loss: 0.8619 validation: 0.950000\n",
            "Iter-1300 loss: 0.7496 validation: 0.950000\n",
            "Iter-1400 loss: 0.8444 validation: 0.950000\n",
            "Iter-1500 loss: 0.7356 validation: 0.950000\n",
            "Iter-1600 loss: 0.6936 validation: 0.950000\n",
            "Iter-1700 loss: 0.8805 validation: 0.950000\n",
            "Iter-1800 loss: 0.8515 validation: 0.950000\n",
            "Iter-1900 loss: 0.7741 validation: 0.950000\n",
            "Iter-2000 loss: 0.7119 validation: 0.950000\n",
            "Iter-2100 loss: 0.7012 validation: 0.950000\n",
            "Iter-2200 loss: 0.6598 validation: 0.950000\n",
            "Iter-2300 loss: 0.6753 validation: 0.950000\n",
            "Iter-2400 loss: 0.7750 validation: 0.950000\n",
            "Iter-2500 loss: 0.6176 validation: 0.950000\n",
            "Iter-2600 loss: 0.6833 validation: 0.950000\n",
            "Iter-2700 loss: 0.5740 validation: 0.950000\n",
            "Iter-2800 loss: 0.6860 validation: 0.950000\n",
            "Iter-2900 loss: 0.6462 validation: 0.950000\n",
            "Iter-3000 loss: 0.5180 validation: 0.950000\n",
            "Iter-3100 loss: 0.7113 validation: 0.950000\n",
            "Iter-3200 loss: 0.5956 validation: 0.950000\n",
            "Iter-3300 loss: 0.6710 validation: 0.950000\n",
            "Iter-3400 loss: 0.6256 validation: 0.950000\n",
            "Iter-3500 loss: 0.5774 validation: 0.950000\n",
            "Iter-3600 loss: 0.5663 validation: 0.960000\n",
            "Iter-3700 loss: 0.5773 validation: 0.950000\n",
            "Iter-3800 loss: 0.5690 validation: 0.960000\n",
            "Iter-3900 loss: 0.6799 validation: 0.960000\n",
            "Iter-4000 loss: 0.5395 validation: 0.950000\n",
            "Iter-4100 loss: 0.5749 validation: 0.950000\n",
            "Iter-4200 loss: 0.5806 validation: 0.960000\n",
            "Iter-4300 loss: 0.5430 validation: 0.950000\n",
            "Iter-4400 loss: 0.6122 validation: 0.950000\n",
            "Iter-4500 loss: 0.5376 validation: 0.960000\n",
            "Iter-4600 loss: 0.5446 validation: 0.960000\n",
            "Iter-4700 loss: 0.5463 validation: 0.950000\n",
            "Iter-4800 loss: 0.4898 validation: 0.960000\n",
            "Iter-4900 loss: 0.5477 validation: 0.960000\n",
            "Iter-5000 loss: 0.6204 validation: 0.960000\n",
            "Iter-5100 loss: 0.4775 validation: 0.960000\n",
            "Iter-5200 loss: 0.4265 validation: 0.960000\n",
            "Iter-5300 loss: 0.4736 validation: 0.960000\n",
            "Iter-5400 loss: 0.5710 validation: 0.960000\n",
            "Iter-5500 loss: 0.5408 validation: 0.960000\n",
            "Iter-5600 loss: 0.5308 validation: 0.960000\n",
            "Iter-5700 loss: 0.4891 validation: 0.950000\n",
            "Iter-5800 loss: 0.5591 validation: 0.960000\n",
            "Iter-5900 loss: 0.4976 validation: 0.960000\n",
            "Iter-6000 loss: 0.4244 validation: 0.960000\n",
            "Iter-6100 loss: 0.5834 validation: 0.970000\n",
            "Iter-6200 loss: 0.4807 validation: 0.960000\n",
            "Iter-6300 loss: 0.4025 validation: 0.970000\n",
            "Iter-6400 loss: 0.5592 validation: 0.960000\n",
            "Iter-6500 loss: 0.4761 validation: 0.960000\n",
            "Iter-6600 loss: 0.3767 validation: 0.970000\n",
            "Iter-6700 loss: 0.4498 validation: 0.960000\n",
            "Iter-6800 loss: 0.4523 validation: 0.970000\n",
            "Iter-6900 loss: 0.4396 validation: 0.970000\n",
            "Iter-7000 loss: 0.5024 validation: 0.960000\n",
            "Iter-7100 loss: 0.5238 validation: 0.960000\n",
            "Iter-7200 loss: 0.4214 validation: 0.970000\n",
            "Iter-7300 loss: 0.4653 validation: 0.960000\n",
            "Iter-7400 loss: 0.4231 validation: 0.960000\n",
            "Iter-7500 loss: 0.4424 validation: 0.960000\n",
            "Iter-7600 loss: 0.4790 validation: 0.970000\n",
            "Iter-7700 loss: 0.4681 validation: 0.960000\n",
            "Iter-7800 loss: 0.4950 validation: 0.960000\n",
            "Iter-7900 loss: 0.5521 validation: 0.970000\n",
            "Iter-8000 loss: 0.4051 validation: 0.970000\n",
            "Iter-8100 loss: 0.4605 validation: 0.970000\n",
            "Iter-8200 loss: 0.4004 validation: 0.960000\n",
            "Iter-8300 loss: 0.4984 validation: 0.970000\n",
            "Iter-8400 loss: 0.4525 validation: 0.970000\n",
            "Iter-8500 loss: 0.4629 validation: 0.970000\n",
            "Iter-8600 loss: 0.3774 validation: 0.970000\n",
            "Iter-8700 loss: 0.3662 validation: 0.970000\n",
            "Iter-8800 loss: 0.4947 validation: 0.970000\n",
            "Iter-8900 loss: 0.4008 validation: 0.970000\n",
            "Iter-9000 loss: 0.3787 validation: 0.970000\n",
            "Iter-9100 loss: 0.4623 validation: 0.970000\n",
            "Iter-9200 loss: 0.3765 validation: 0.970000\n",
            "Iter-9300 loss: 0.3452 validation: 0.970000\n",
            "Iter-9400 loss: 0.4412 validation: 0.970000\n",
            "Iter-9500 loss: 0.4725 validation: 0.970000\n",
            "Iter-9600 loss: 0.3844 validation: 0.970000\n",
            "Iter-9700 loss: 0.6043 validation: 0.970000\n",
            "Iter-9800 loss: 0.4542 validation: 0.970000\n",
            "Iter-9900 loss: 0.3518 validation: 0.970000\n",
            "Iter-10000 loss: 0.5114 validation: 0.970000\n",
            "Iter-10100 loss: 0.3775 validation: 0.970000\n",
            "Iter-10200 loss: 0.3221 validation: 0.970000\n",
            "Iter-10300 loss: 0.3331 validation: 0.970000\n",
            "Iter-10400 loss: 0.3240 validation: 0.970000\n",
            "Iter-10500 loss: 0.3326 validation: 0.970000\n",
            "Iter-10600 loss: 0.3331 validation: 0.970000\n",
            "Iter-10700 loss: 0.3923 validation: 0.970000\n",
            "Iter-10800 loss: 0.3485 validation: 0.970000\n",
            "Iter-10900 loss: 0.4094 validation: 0.970000\n",
            "Iter-11000 loss: 0.3770 validation: 0.970000\n",
            "Iter-11100 loss: 0.2915 validation: 0.970000\n",
            "Iter-11200 loss: 0.3713 validation: 0.970000\n",
            "Iter-11300 loss: 0.3601 validation: 0.970000\n",
            "Iter-11400 loss: 0.3820 validation: 0.970000\n",
            "Iter-11500 loss: 0.4102 validation: 0.970000\n",
            "Iter-11600 loss: 0.3796 validation: 0.970000\n",
            "Iter-11700 loss: 0.3436 validation: 0.970000\n",
            "Iter-11800 loss: 0.3893 validation: 0.970000\n",
            "Iter-11900 loss: 0.3936 validation: 0.970000\n",
            "Iter-12000 loss: 0.3491 validation: 0.970000\n",
            "Iter-12100 loss: 0.3518 validation: 0.970000\n",
            "Iter-12200 loss: 0.3261 validation: 0.970000\n",
            "Iter-12300 loss: 0.3660 validation: 0.970000\n",
            "Iter-12400 loss: 0.4001 validation: 0.970000\n",
            "Iter-12500 loss: 0.2807 validation: 0.970000\n",
            "Iter-12600 loss: 0.3358 validation: 0.970000\n",
            "Iter-12700 loss: 0.3100 validation: 0.970000\n",
            "Iter-12800 loss: 0.3566 validation: 0.970000\n",
            "Iter-12900 loss: 0.3919 validation: 0.970000\n",
            "Iter-13000 loss: 0.2792 validation: 0.970000\n",
            "Iter-13100 loss: 0.3053 validation: 0.970000\n",
            "Iter-13200 loss: 0.3118 validation: 0.970000\n",
            "Iter-13300 loss: 0.3867 validation: 0.970000\n",
            "Iter-13400 loss: 0.3732 validation: 0.970000\n",
            "Iter-13500 loss: 0.2627 validation: 0.970000\n",
            "Iter-13600 loss: 0.3131 validation: 0.970000\n",
            "Iter-13700 loss: 0.2504 validation: 0.970000\n",
            "Iter-13800 loss: 0.4268 validation: 0.970000\n",
            "Iter-13900 loss: 0.3272 validation: 0.970000\n",
            "Iter-14000 loss: 0.3545 validation: 0.970000\n",
            "Iter-14100 loss: 0.3015 validation: 0.970000\n",
            "Iter-14200 loss: 0.3506 validation: 0.970000\n",
            "Iter-14300 loss: 0.3993 validation: 0.970000\n",
            "Iter-14400 loss: 0.4415 validation: 0.970000\n",
            "Iter-14500 loss: 0.3220 validation: 0.970000\n",
            "Iter-14600 loss: 0.2552 validation: 0.970000\n",
            "Iter-14700 loss: 0.3514 validation: 0.970000\n",
            "Iter-14800 loss: 0.2894 validation: 0.970000\n",
            "Iter-14900 loss: 0.2794 validation: 0.970000\n",
            "Iter-15000 loss: 0.3071 validation: 0.970000\n",
            "Iter-15100 loss: 0.3156 validation: 0.970000\n",
            "Iter-15200 loss: 0.2162 validation: 0.970000\n",
            "Iter-15300 loss: 0.3174 validation: 0.970000\n",
            "Iter-15400 loss: 0.3515 validation: 0.970000\n",
            "Iter-15500 loss: 0.2859 validation: 0.970000\n",
            "Iter-15600 loss: 0.3907 validation: 0.970000\n",
            "Iter-15700 loss: 0.3029 validation: 0.970000\n",
            "Iter-15800 loss: 0.2882 validation: 0.970000\n",
            "Iter-15900 loss: 0.2979 validation: 0.970000\n",
            "Iter-16000 loss: 0.3173 validation: 0.970000\n",
            "Iter-16100 loss: 0.3833 validation: 0.970000\n",
            "Iter-16200 loss: 0.2802 validation: 0.970000\n",
            "Iter-16300 loss: 0.3469 validation: 0.970000\n",
            "Iter-16400 loss: 0.3008 validation: 0.970000\n",
            "Iter-16500 loss: 0.3964 validation: 0.970000\n",
            "Iter-16600 loss: 0.3144 validation: 0.970000\n",
            "Iter-16700 loss: 0.3562 validation: 0.970000\n",
            "Iter-16800 loss: 0.2634 validation: 0.970000\n",
            "Iter-16900 loss: 0.2234 validation: 0.970000\n",
            "Iter-17000 loss: 0.4045 validation: 0.970000\n",
            "Iter-17100 loss: 0.2011 validation: 0.970000\n",
            "Iter-17200 loss: 0.2396 validation: 0.970000\n",
            "Iter-17300 loss: 0.3067 validation: 0.970000\n",
            "Iter-17400 loss: 0.3549 validation: 0.970000\n",
            "Iter-17500 loss: 0.2105 validation: 0.970000\n",
            "Iter-17600 loss: 0.4180 validation: 0.970000\n",
            "Iter-17700 loss: 0.2543 validation: 0.970000\n",
            "Iter-17800 loss: 0.2911 validation: 0.970000\n",
            "Iter-17900 loss: 0.2079 validation: 0.970000\n",
            "Iter-18000 loss: 0.2176 validation: 0.970000\n",
            "Iter-18100 loss: 0.2827 validation: 0.970000\n",
            "Iter-18200 loss: 0.2087 validation: 0.970000\n",
            "Iter-18300 loss: 0.3114 validation: 0.970000\n",
            "Iter-18400 loss: 0.2808 validation: 0.970000\n",
            "Iter-18500 loss: 0.2649 validation: 0.970000\n",
            "Iter-18600 loss: 0.2361 validation: 0.970000\n",
            "Iter-18700 loss: 0.2967 validation: 0.970000\n",
            "Iter-18800 loss: 0.2456 validation: 0.970000\n",
            "Iter-18900 loss: 0.2783 validation: 0.970000\n",
            "Iter-19000 loss: 0.2508 validation: 0.970000\n",
            "Iter-19100 loss: 0.2237 validation: 0.970000\n",
            "Iter-19200 loss: 0.3081 validation: 0.970000\n",
            "Iter-19300 loss: 0.2239 validation: 0.970000\n",
            "Iter-19400 loss: 0.2648 validation: 0.970000\n",
            "Iter-19500 loss: 0.1899 validation: 0.970000\n",
            "Iter-19600 loss: 0.2431 validation: 0.970000\n",
            "Iter-19700 loss: 0.2072 validation: 0.970000\n",
            "Iter-19800 loss: 0.3564 validation: 0.970000\n",
            "Iter-19900 loss: 0.2683 validation: 0.970000\n",
            "Iter-20000 loss: 0.2388 validation: 0.970000\n",
            "Iter-20100 loss: 0.2526 validation: 0.970000\n",
            "Iter-20200 loss: 0.2856 validation: 0.970000\n",
            "Iter-20300 loss: 0.2683 validation: 0.970000\n",
            "Iter-20400 loss: 0.2355 validation: 0.970000\n",
            "Iter-20500 loss: 0.2052 validation: 0.970000\n",
            "Iter-20600 loss: 0.2395 validation: 0.970000\n",
            "Iter-20700 loss: 0.3176 validation: 0.970000\n",
            "Iter-20800 loss: 0.2348 validation: 0.970000\n",
            "Iter-20900 loss: 0.2646 validation: 0.970000\n",
            "Iter-21000 loss: 0.2226 validation: 0.970000\n",
            "Iter-21100 loss: 0.2726 validation: 0.970000\n",
            "Iter-21200 loss: 0.2462 validation: 0.970000\n",
            "Iter-21300 loss: 0.2877 validation: 0.970000\n",
            "Iter-21400 loss: 0.2352 validation: 0.970000\n",
            "Iter-21500 loss: 0.2793 validation: 0.970000\n",
            "Iter-21600 loss: 0.3302 validation: 0.970000\n",
            "Iter-21700 loss: 0.2492 validation: 0.970000\n",
            "Iter-21800 loss: 0.2609 validation: 0.970000\n",
            "Iter-21900 loss: 0.2884 validation: 0.970000\n",
            "Iter-22000 loss: 0.2722 validation: 0.970000\n",
            "Iter-22100 loss: 0.2610 validation: 0.970000\n",
            "Iter-22200 loss: 0.2570 validation: 0.970000\n",
            "Iter-22300 loss: 0.2325 validation: 0.970000\n",
            "Iter-22400 loss: 0.2435 validation: 0.970000\n",
            "Iter-22500 loss: 0.2336 validation: 0.970000\n",
            "Iter-22600 loss: 0.1709 validation: 0.970000\n",
            "Iter-22700 loss: 0.2696 validation: 0.970000\n",
            "Iter-22800 loss: 0.3220 validation: 0.970000\n",
            "Iter-22900 loss: 0.1989 validation: 0.970000\n",
            "Iter-23000 loss: 0.2372 validation: 0.970000\n",
            "Iter-23100 loss: 0.1963 validation: 0.970000\n",
            "Iter-23200 loss: 0.2252 validation: 0.970000\n",
            "Iter-23300 loss: 0.2205 validation: 0.970000\n",
            "Iter-23400 loss: 0.2047 validation: 0.970000\n",
            "Iter-23500 loss: 0.3209 validation: 0.970000\n",
            "Iter-23600 loss: 0.2265 validation: 0.970000\n",
            "Iter-23700 loss: 0.1877 validation: 0.970000\n",
            "Iter-23800 loss: 0.2368 validation: 0.970000\n",
            "Iter-23900 loss: 0.2337 validation: 0.970000\n",
            "Iter-24000 loss: 0.2213 validation: 0.970000\n",
            "Iter-24100 loss: 0.2691 validation: 0.970000\n",
            "Iter-24200 loss: 0.1770 validation: 0.970000\n",
            "Iter-24300 loss: 0.2430 validation: 0.970000\n",
            "Iter-24400 loss: 0.1975 validation: 0.970000\n",
            "Iter-24500 loss: 0.2375 validation: 0.970000\n",
            "Iter-24600 loss: 0.1959 validation: 0.970000\n",
            "Iter-24700 loss: 0.2189 validation: 0.970000\n",
            "Iter-24800 loss: 0.1974 validation: 0.970000\n",
            "Iter-24900 loss: 0.3011 validation: 0.970000\n",
            "Iter-25000 loss: 0.2785 validation: 0.970000\n",
            "Iter-25100 loss: 0.3439 validation: 0.970000\n",
            "Iter-25200 loss: 0.2356 validation: 0.970000\n",
            "Iter-25300 loss: 0.2425 validation: 0.970000\n",
            "Iter-25400 loss: 0.2395 validation: 0.970000\n",
            "Iter-25500 loss: 0.1776 validation: 0.970000\n",
            "Iter-25600 loss: 0.2716 validation: 0.970000\n",
            "Iter-25700 loss: 0.1797 validation: 0.970000\n",
            "Iter-25800 loss: 0.2403 validation: 0.970000\n",
            "Iter-25900 loss: 0.2126 validation: 0.970000\n",
            "Iter-26000 loss: 0.2043 validation: 0.970000\n",
            "Iter-26100 loss: 0.1757 validation: 0.970000\n",
            "Iter-26200 loss: 0.3275 validation: 0.970000\n",
            "Iter-26300 loss: 0.3020 validation: 0.970000\n",
            "Iter-26400 loss: 0.2268 validation: 0.970000\n",
            "Iter-26500 loss: 0.3119 validation: 0.970000\n",
            "Iter-26600 loss: 0.2372 validation: 0.970000\n",
            "Iter-26700 loss: 0.2126 validation: 0.970000\n",
            "Iter-26800 loss: 0.1975 validation: 0.970000\n",
            "Iter-26900 loss: 0.2279 validation: 0.970000\n",
            "Iter-27000 loss: 0.2105 validation: 0.970000\n",
            "Iter-27100 loss: 0.1991 validation: 0.970000\n",
            "Iter-27200 loss: 0.2840 validation: 0.970000\n",
            "Iter-27300 loss: 0.2180 validation: 0.970000\n",
            "Iter-27400 loss: 0.2458 validation: 0.970000\n",
            "Iter-27500 loss: 0.2325 validation: 0.970000\n",
            "Iter-27600 loss: 0.2298 validation: 0.970000\n",
            "Iter-27700 loss: 0.2405 validation: 0.970000\n",
            "Iter-27800 loss: 0.2023 validation: 0.970000\n",
            "Iter-27900 loss: 0.3288 validation: 0.970000\n",
            "Iter-28000 loss: 0.2385 validation: 0.970000\n",
            "Iter-28100 loss: 0.1934 validation: 0.970000\n",
            "Iter-28200 loss: 0.1582 validation: 0.970000\n",
            "Iter-28300 loss: 0.2142 validation: 0.970000\n",
            "Iter-28400 loss: 0.2631 validation: 0.970000\n",
            "Iter-28500 loss: 0.2428 validation: 0.970000\n",
            "Iter-28600 loss: 0.1898 validation: 0.970000\n",
            "Iter-28700 loss: 0.1882 validation: 0.970000\n",
            "Iter-28800 loss: 0.2484 validation: 0.970000\n",
            "Iter-28900 loss: 0.1939 validation: 0.970000\n",
            "Iter-29000 loss: 0.2361 validation: 0.970000\n",
            "Iter-29100 loss: 0.2650 validation: 0.970000\n",
            "Iter-29200 loss: 0.1829 validation: 0.970000\n",
            "Iter-29300 loss: 0.1576 validation: 0.970000\n",
            "Iter-29400 loss: 0.1960 validation: 0.970000\n",
            "Iter-29500 loss: 0.1665 validation: 0.970000\n",
            "Iter-29600 loss: 0.2228 validation: 0.970000\n",
            "Iter-29700 loss: 0.2672 validation: 0.970000\n",
            "Iter-29800 loss: 0.2050 validation: 0.970000\n",
            "Iter-29900 loss: 0.1949 validation: 0.970000\n",
            "Iter-30000 loss: 0.1762 validation: 0.970000\n",
            "Iter-30100 loss: 0.1646 validation: 0.970000\n",
            "Iter-30200 loss: 0.2024 validation: 0.970000\n",
            "Iter-30300 loss: 0.1614 validation: 0.970000\n",
            "Iter-30400 loss: 0.2179 validation: 0.970000\n",
            "Iter-30500 loss: 0.2107 validation: 0.970000\n",
            "Iter-30600 loss: 0.2205 validation: 0.980000\n",
            "Iter-30700 loss: 0.1858 validation: 0.970000\n",
            "Iter-30800 loss: 0.1783 validation: 0.970000\n",
            "Iter-30900 loss: 0.1632 validation: 0.980000\n",
            "Iter-31000 loss: 0.1768 validation: 0.970000\n",
            "Iter-31100 loss: 0.1835 validation: 0.980000\n",
            "Iter-31200 loss: 0.2306 validation: 0.970000\n",
            "Iter-31300 loss: 0.1751 validation: 0.980000\n",
            "Iter-31400 loss: 0.1814 validation: 0.970000\n",
            "Iter-31500 loss: 0.2311 validation: 0.980000\n",
            "Iter-31600 loss: 0.1700 validation: 0.980000\n",
            "Iter-31700 loss: 0.2695 validation: 0.980000\n",
            "Iter-31800 loss: 0.1751 validation: 0.970000\n",
            "Iter-31900 loss: 0.1879 validation: 0.980000\n",
            "Iter-32000 loss: 0.2808 validation: 0.980000\n",
            "Iter-32100 loss: 0.1821 validation: 0.980000\n",
            "Iter-32200 loss: 0.2165 validation: 0.980000\n",
            "Iter-32300 loss: 0.2964 validation: 0.980000\n",
            "Iter-32400 loss: 0.2097 validation: 0.980000\n",
            "Iter-32500 loss: 0.1909 validation: 0.970000\n",
            "Iter-32600 loss: 0.2054 validation: 0.970000\n",
            "Iter-32700 loss: 0.2236 validation: 0.970000\n",
            "Iter-32800 loss: 0.1380 validation: 0.970000\n",
            "Iter-32900 loss: 0.2014 validation: 0.970000\n",
            "Iter-33000 loss: 0.1952 validation: 0.970000\n",
            "Iter-33100 loss: 0.1876 validation: 0.980000\n",
            "Iter-33200 loss: 0.2018 validation: 0.970000\n",
            "Iter-33300 loss: 0.2154 validation: 0.970000\n",
            "Iter-33400 loss: 0.2523 validation: 0.970000\n",
            "Iter-33500 loss: 0.3188 validation: 0.970000\n",
            "Iter-33600 loss: 0.2036 validation: 0.980000\n",
            "Iter-33700 loss: 0.1524 validation: 0.970000\n",
            "Iter-33800 loss: 0.2114 validation: 0.970000\n",
            "Iter-33900 loss: 0.1953 validation: 0.970000\n",
            "Iter-34000 loss: 0.1950 validation: 0.970000\n",
            "Iter-34100 loss: 0.1997 validation: 0.970000\n",
            "Iter-34200 loss: 0.2336 validation: 0.970000\n",
            "Iter-34300 loss: 0.2102 validation: 0.970000\n",
            "Iter-34400 loss: 0.2725 validation: 0.970000\n",
            "Iter-34500 loss: 0.1971 validation: 0.970000\n",
            "Iter-34600 loss: 0.1831 validation: 0.980000\n",
            "Iter-34700 loss: 0.1412 validation: 0.970000\n",
            "Iter-34800 loss: 0.1834 validation: 0.970000\n",
            "Iter-34900 loss: 0.2662 validation: 0.970000\n",
            "Iter-35000 loss: 0.1643 validation: 0.980000\n",
            "Iter-35100 loss: 0.2156 validation: 0.970000\n",
            "Iter-35200 loss: 0.2031 validation: 0.970000\n",
            "Iter-35300 loss: 0.1569 validation: 0.980000\n",
            "Iter-35400 loss: 0.2922 validation: 0.980000\n",
            "Iter-35500 loss: 0.1753 validation: 0.980000\n",
            "Iter-35600 loss: 0.2376 validation: 0.970000\n",
            "Iter-35700 loss: 0.1562 validation: 0.980000\n",
            "Iter-35800 loss: 0.2023 validation: 0.970000\n",
            "Iter-35900 loss: 0.1917 validation: 0.980000\n",
            "Iter-36000 loss: 0.1757 validation: 0.980000\n",
            "Iter-36100 loss: 0.2000 validation: 0.980000\n",
            "Iter-36200 loss: 0.2645 validation: 0.970000\n",
            "Iter-36300 loss: 0.1867 validation: 0.980000\n",
            "Iter-36400 loss: 0.2201 validation: 0.980000\n",
            "Iter-36500 loss: 0.2358 validation: 0.980000\n",
            "Iter-36600 loss: 0.1643 validation: 0.980000\n",
            "Iter-36700 loss: 0.2185 validation: 0.970000\n",
            "Iter-36800 loss: 0.2254 validation: 0.970000\n",
            "Iter-36900 loss: 0.2232 validation: 0.970000\n",
            "Iter-37000 loss: 0.1488 validation: 0.970000\n",
            "Iter-37100 loss: 0.1815 validation: 0.970000\n",
            "Iter-37200 loss: 0.2272 validation: 0.970000\n",
            "Iter-37300 loss: 0.2269 validation: 0.980000\n",
            "Iter-37400 loss: 0.1417 validation: 0.970000\n",
            "Iter-37500 loss: 0.1981 validation: 0.970000\n",
            "Iter-37600 loss: 0.1395 validation: 0.970000\n",
            "Iter-37700 loss: 0.1838 validation: 0.980000\n",
            "Iter-37800 loss: 0.1833 validation: 0.970000\n",
            "Iter-37900 loss: 0.1750 validation: 0.970000\n",
            "Iter-38000 loss: 0.2412 validation: 0.980000\n",
            "Iter-38100 loss: 0.2723 validation: 0.980000\n",
            "Iter-38200 loss: 0.1778 validation: 0.980000\n",
            "Iter-38300 loss: 0.2003 validation: 0.970000\n",
            "Iter-38400 loss: 0.1781 validation: 0.980000\n",
            "Iter-38500 loss: 0.1977 validation: 0.980000\n",
            "Iter-38600 loss: 0.1728 validation: 0.970000\n",
            "Iter-38700 loss: 0.1668 validation: 0.970000\n",
            "Iter-38800 loss: 0.2594 validation: 0.980000\n",
            "Iter-38900 loss: 0.1737 validation: 0.970000\n",
            "Iter-39000 loss: 0.1971 validation: 0.970000\n",
            "Iter-39100 loss: 0.2407 validation: 0.970000\n",
            "Iter-39200 loss: 0.2005 validation: 0.970000\n",
            "Iter-39300 loss: 0.1622 validation: 0.980000\n",
            "Iter-39400 loss: 0.1585 validation: 0.970000\n",
            "Iter-39500 loss: 0.1866 validation: 0.980000\n",
            "Iter-39600 loss: 0.1585 validation: 0.970000\n",
            "Iter-39700 loss: 0.1941 validation: 0.980000\n",
            "Iter-39800 loss: 0.1681 validation: 0.970000\n",
            "Iter-39900 loss: 0.2164 validation: 0.980000\n",
            "Iter-40000 loss: 0.1750 validation: 0.980000\n",
            "Iter-40100 loss: 0.1482 validation: 0.970000\n",
            "Iter-40200 loss: 0.1210 validation: 0.980000\n",
            "Iter-40300 loss: 0.1761 validation: 0.970000\n",
            "Iter-40400 loss: 0.2239 validation: 0.980000\n",
            "Iter-40500 loss: 0.2352 validation: 0.980000\n",
            "Iter-40600 loss: 0.1752 validation: 0.980000\n",
            "Iter-40700 loss: 0.1998 validation: 0.980000\n",
            "Iter-40800 loss: 0.1894 validation: 0.980000\n",
            "Iter-40900 loss: 0.2085 validation: 0.980000\n",
            "Iter-41000 loss: 0.1792 validation: 0.970000\n",
            "Iter-41100 loss: 0.1979 validation: 0.970000\n",
            "Iter-41200 loss: 0.2465 validation: 0.980000\n",
            "Iter-41300 loss: 0.1764 validation: 0.970000\n",
            "Iter-41400 loss: 0.1695 validation: 0.980000\n",
            "Iter-41500 loss: 0.1752 validation: 0.970000\n",
            "Iter-41600 loss: 0.1504 validation: 0.970000\n",
            "Iter-41700 loss: 0.1976 validation: 0.970000\n",
            "Iter-41800 loss: 0.1957 validation: 0.970000\n",
            "Iter-41900 loss: 0.1810 validation: 0.970000\n",
            "Iter-42000 loss: 0.2395 validation: 0.980000\n",
            "Iter-42100 loss: 0.2658 validation: 0.970000\n",
            "Iter-42200 loss: 0.2089 validation: 0.970000\n",
            "Iter-42300 loss: 0.1609 validation: 0.970000\n",
            "Iter-42400 loss: 0.1922 validation: 0.970000\n",
            "Iter-42500 loss: 0.1683 validation: 0.970000\n",
            "Iter-42600 loss: 0.1442 validation: 0.970000\n",
            "Iter-42700 loss: 0.1951 validation: 0.970000\n",
            "Iter-42800 loss: 0.1806 validation: 0.980000\n",
            "Iter-42900 loss: 0.1489 validation: 0.980000\n",
            "Iter-43000 loss: 0.1636 validation: 0.970000\n",
            "Iter-43100 loss: 0.1699 validation: 0.970000\n",
            "Iter-43200 loss: 0.1409 validation: 0.980000\n",
            "Iter-43300 loss: 0.1543 validation: 0.970000\n",
            "Iter-43400 loss: 0.1878 validation: 0.970000\n",
            "Iter-43500 loss: 0.1515 validation: 0.970000\n",
            "Iter-43600 loss: 0.1477 validation: 0.970000\n",
            "Iter-43700 loss: 0.2211 validation: 0.980000\n",
            "Iter-43800 loss: 0.1971 validation: 0.980000\n",
            "Iter-43900 loss: 0.2124 validation: 0.970000\n",
            "Iter-44000 loss: 0.2044 validation: 0.980000\n",
            "Iter-44100 loss: 0.1762 validation: 0.980000\n",
            "Iter-44200 loss: 0.1857 validation: 0.980000\n",
            "Iter-44300 loss: 0.1998 validation: 0.980000\n",
            "Iter-44400 loss: 0.1797 validation: 0.970000\n",
            "Iter-44500 loss: 0.1975 validation: 0.970000\n",
            "Iter-44600 loss: 0.1255 validation: 0.980000\n",
            "Iter-44700 loss: 0.1867 validation: 0.980000\n",
            "Iter-44800 loss: 0.1810 validation: 0.980000\n",
            "Iter-44900 loss: 0.1914 validation: 0.980000\n",
            "Iter-45000 loss: 0.1793 validation: 0.980000\n",
            "Iter-45100 loss: 0.1580 validation: 0.970000\n",
            "Iter-45200 loss: 0.1685 validation: 0.980000\n",
            "Iter-45300 loss: 0.1844 validation: 0.970000\n",
            "Iter-45400 loss: 0.1846 validation: 0.980000\n",
            "Iter-45500 loss: 0.1815 validation: 0.980000\n",
            "Iter-45600 loss: 0.1686 validation: 0.980000\n",
            "Iter-45700 loss: 0.1551 validation: 0.980000\n",
            "Iter-45800 loss: 0.1716 validation: 0.980000\n",
            "Iter-45900 loss: 0.1251 validation: 0.980000\n",
            "Iter-46000 loss: 0.1585 validation: 0.980000\n",
            "Iter-46100 loss: 0.1557 validation: 0.980000\n",
            "Iter-46200 loss: 0.1493 validation: 0.980000\n",
            "Iter-46300 loss: 0.1558 validation: 0.980000\n",
            "Iter-46400 loss: 0.1800 validation: 0.980000\n",
            "Iter-46500 loss: 0.1591 validation: 0.980000\n",
            "Iter-46600 loss: 0.1435 validation: 0.980000\n",
            "Iter-46700 loss: 0.1882 validation: 0.980000\n",
            "Iter-46800 loss: 0.1769 validation: 0.980000\n",
            "Iter-46900 loss: 0.1563 validation: 0.980000\n",
            "Iter-47000 loss: 0.1890 validation: 0.980000\n",
            "Iter-47100 loss: 0.1712 validation: 0.970000\n",
            "Iter-47200 loss: 0.1747 validation: 0.970000\n",
            "Iter-47300 loss: 0.1459 validation: 0.980000\n",
            "Iter-47400 loss: 0.1726 validation: 0.980000\n",
            "Iter-47500 loss: 0.1707 validation: 0.970000\n",
            "Iter-47600 loss: 0.1659 validation: 0.980000\n",
            "Iter-47700 loss: 0.1926 validation: 0.980000\n",
            "Iter-47800 loss: 0.1583 validation: 0.970000\n",
            "Iter-47900 loss: 0.2010 validation: 0.970000\n",
            "Iter-48000 loss: 0.2040 validation: 0.980000\n",
            "Iter-48100 loss: 0.2142 validation: 0.980000\n",
            "Iter-48200 loss: 0.2453 validation: 0.980000\n",
            "Iter-48300 loss: 0.1444 validation: 0.980000\n",
            "Iter-48400 loss: 0.1640 validation: 0.970000\n",
            "Iter-48500 loss: 0.1390 validation: 0.970000\n",
            "Iter-48600 loss: 0.1580 validation: 0.980000\n",
            "Iter-48700 loss: 0.1461 validation: 0.980000\n",
            "Iter-48800 loss: 0.1759 validation: 0.970000\n",
            "Iter-48900 loss: 0.1753 validation: 0.970000\n",
            "Iter-49000 loss: 0.1469 validation: 0.970000\n",
            "Iter-49100 loss: 0.1295 validation: 0.980000\n",
            "Iter-49200 loss: 0.1496 validation: 0.970000\n",
            "Iter-49300 loss: 0.1682 validation: 0.970000\n",
            "Iter-49400 loss: 0.1632 validation: 0.970000\n",
            "Iter-49500 loss: 0.1536 validation: 0.980000\n",
            "Iter-49600 loss: 0.2290 validation: 0.970000\n",
            "Iter-49700 loss: 0.1621 validation: 0.980000\n",
            "Iter-49800 loss: 0.1868 validation: 0.970000\n",
            "Iter-49900 loss: 0.1978 validation: 0.970000\n",
            "Iter-50000 loss: 0.1490 validation: 0.970000\n",
            "Iter-50100 loss: 0.1702 validation: 0.980000\n",
            "Iter-50200 loss: 0.1553 validation: 0.980000\n",
            "Iter-50300 loss: 0.1639 validation: 0.980000\n",
            "Iter-50400 loss: 0.1920 validation: 0.980000\n",
            "Iter-50500 loss: 0.1754 validation: 0.980000\n",
            "Iter-50600 loss: 0.2027 validation: 0.980000\n",
            "Iter-50700 loss: 0.2068 validation: 0.980000\n",
            "Iter-50800 loss: 0.1353 validation: 0.980000\n",
            "Iter-50900 loss: 0.1649 validation: 0.980000\n",
            "Iter-51000 loss: 0.1578 validation: 0.980000\n",
            "Iter-51100 loss: 0.2083 validation: 0.980000\n",
            "Iter-51200 loss: 0.1474 validation: 0.980000\n",
            "Iter-51300 loss: 0.1706 validation: 0.980000\n",
            "Iter-51400 loss: 0.2150 validation: 0.970000\n",
            "Iter-51500 loss: 0.1430 validation: 0.980000\n",
            "Iter-51600 loss: 0.1838 validation: 0.980000\n",
            "Iter-51700 loss: 0.1886 validation: 0.980000\n",
            "Iter-51800 loss: 0.1694 validation: 0.980000\n",
            "Iter-51900 loss: 0.1540 validation: 0.980000\n",
            "Iter-52000 loss: 0.2152 validation: 0.980000\n",
            "Iter-52100 loss: 0.1556 validation: 0.980000\n",
            "Iter-52200 loss: 0.1829 validation: 0.980000\n",
            "Iter-52300 loss: 0.1580 validation: 0.980000\n",
            "Iter-52400 loss: 0.1393 validation: 0.980000\n",
            "Iter-52500 loss: 0.1260 validation: 0.970000\n",
            "Iter-52600 loss: 0.1809 validation: 0.980000\n",
            "Iter-52700 loss: 0.1648 validation: 0.980000\n",
            "Iter-52800 loss: 0.1118 validation: 0.980000\n",
            "Iter-52900 loss: 0.1408 validation: 0.970000\n",
            "Iter-53000 loss: 0.1675 validation: 0.980000\n",
            "Iter-53100 loss: 0.1921 validation: 0.980000\n",
            "Iter-53200 loss: 0.1833 validation: 0.970000\n",
            "Iter-53300 loss: 0.1507 validation: 0.970000\n",
            "Iter-53400 loss: 0.1304 validation: 0.970000\n",
            "Iter-53500 loss: 0.1746 validation: 0.970000\n",
            "Iter-53600 loss: 0.1649 validation: 0.970000\n",
            "Iter-53700 loss: 0.1994 validation: 0.970000\n",
            "Iter-53800 loss: 0.1425 validation: 0.980000\n",
            "Iter-53900 loss: 0.1767 validation: 0.980000\n",
            "Iter-54000 loss: 0.1667 validation: 0.970000\n",
            "Iter-54100 loss: 0.1905 validation: 0.980000\n",
            "Iter-54200 loss: 0.1474 validation: 0.970000\n",
            "Iter-54300 loss: 0.1906 validation: 0.980000\n",
            "Iter-54400 loss: 0.1600 validation: 0.970000\n",
            "Iter-54500 loss: 0.2258 validation: 0.980000\n",
            "Iter-54600 loss: 0.1228 validation: 0.980000\n",
            "Iter-54700 loss: 0.1799 validation: 0.980000\n",
            "Iter-54800 loss: 0.1437 validation: 0.980000\n",
            "Iter-54900 loss: 0.1441 validation: 0.970000\n",
            "Iter-55000 loss: 0.1900 validation: 0.970000\n",
            "Iter-55100 loss: 0.1636 validation: 0.970000\n",
            "Iter-55200 loss: 0.1464 validation: 0.970000\n",
            "Iter-55300 loss: 0.1673 validation: 0.970000\n",
            "Iter-55400 loss: 0.1739 validation: 0.980000\n",
            "Iter-55500 loss: 0.1186 validation: 0.970000\n",
            "Iter-55600 loss: 0.1799 validation: 0.980000\n",
            "Iter-55700 loss: 0.1726 validation: 0.980000\n",
            "Iter-55800 loss: 0.1842 validation: 0.980000\n",
            "Iter-55900 loss: 0.2005 validation: 0.980000\n",
            "Iter-56000 loss: 0.1871 validation: 0.970000\n",
            "Iter-56100 loss: 0.1703 validation: 0.980000\n",
            "Iter-56200 loss: 0.2106 validation: 0.980000\n",
            "Iter-56300 loss: 0.1872 validation: 0.980000\n",
            "Iter-56400 loss: 0.1504 validation: 0.980000\n",
            "Iter-56500 loss: 0.2017 validation: 0.980000\n",
            "Iter-56600 loss: 0.1784 validation: 0.980000\n",
            "Iter-56700 loss: 0.1596 validation: 0.980000\n",
            "Iter-56800 loss: 0.1300 validation: 0.980000\n",
            "Iter-56900 loss: 0.1908 validation: 0.980000\n",
            "Iter-57000 loss: 0.1485 validation: 0.980000\n",
            "Iter-57100 loss: 0.1247 validation: 0.980000\n",
            "Iter-57200 loss: 0.1442 validation: 0.980000\n",
            "Iter-57300 loss: 0.1414 validation: 0.980000\n",
            "Iter-57400 loss: 0.1334 validation: 0.980000\n",
            "Iter-57500 loss: 0.1681 validation: 0.980000\n",
            "Iter-57600 loss: 0.1376 validation: 0.980000\n",
            "Iter-57700 loss: 0.1426 validation: 0.980000\n",
            "Iter-57800 loss: 0.1727 validation: 0.980000\n",
            "Iter-57900 loss: 0.1600 validation: 0.980000\n",
            "Iter-58000 loss: 0.1170 validation: 0.980000\n",
            "Iter-58100 loss: 0.1340 validation: 0.980000\n",
            "Iter-58200 loss: 0.1350 validation: 0.980000\n",
            "Iter-58300 loss: 0.2156 validation: 0.980000\n",
            "Iter-58400 loss: 0.1314 validation: 0.980000\n",
            "Iter-58500 loss: 0.1510 validation: 0.980000\n",
            "Iter-58600 loss: 0.1642 validation: 0.980000\n",
            "Iter-58700 loss: 0.1241 validation: 0.980000\n",
            "Iter-58800 loss: 0.1456 validation: 0.980000\n",
            "Iter-58900 loss: 0.1609 validation: 0.980000\n",
            "Iter-59000 loss: 0.1226 validation: 0.980000\n",
            "Iter-59100 loss: 0.1791 validation: 0.980000\n",
            "Iter-59200 loss: 0.1748 validation: 0.980000\n",
            "Iter-59300 loss: 0.1682 validation: 0.980000\n",
            "Iter-59400 loss: 0.1279 validation: 0.980000\n",
            "Iter-59500 loss: 0.1347 validation: 0.980000\n",
            "Iter-59600 loss: 0.1507 validation: 0.980000\n",
            "Iter-59700 loss: 0.1552 validation: 0.980000\n",
            "Iter-59800 loss: 0.1628 validation: 0.980000\n",
            "Iter-59900 loss: 0.1276 validation: 0.980000\n",
            "Iter-60000 loss: 0.1417 validation: 0.980000\n",
            "Iter-60100 loss: 0.1804 validation: 0.980000\n",
            "Iter-60200 loss: 0.1643 validation: 0.980000\n",
            "Iter-60300 loss: 0.1503 validation: 0.980000\n",
            "Iter-60400 loss: 0.1019 validation: 0.980000\n",
            "Iter-60500 loss: 0.1385 validation: 0.980000\n",
            "Iter-60600 loss: 0.1679 validation: 0.980000\n",
            "Iter-60700 loss: 0.1251 validation: 0.980000\n",
            "Iter-60800 loss: 0.1455 validation: 0.980000\n",
            "Iter-60900 loss: 0.1774 validation: 0.980000\n",
            "Iter-61000 loss: 0.1448 validation: 0.980000\n",
            "Iter-61100 loss: 0.1092 validation: 0.980000\n",
            "Iter-61200 loss: 0.2054 validation: 0.970000\n",
            "Iter-61300 loss: 0.1540 validation: 0.980000\n",
            "Iter-61400 loss: 0.1427 validation: 0.980000\n",
            "Iter-61500 loss: 0.1026 validation: 0.980000\n",
            "Iter-61600 loss: 0.1387 validation: 0.980000\n",
            "Iter-61700 loss: 0.2048 validation: 0.980000\n",
            "Iter-61800 loss: 0.1353 validation: 0.980000\n",
            "Iter-61900 loss: 0.1236 validation: 0.980000\n",
            "Iter-62000 loss: 0.1734 validation: 0.980000\n",
            "Iter-62100 loss: 0.1491 validation: 0.980000\n",
            "Iter-62200 loss: 0.1580 validation: 0.980000\n",
            "Iter-62300 loss: 0.1459 validation: 0.980000\n",
            "Iter-62400 loss: 0.1685 validation: 0.980000\n",
            "Iter-62500 loss: 0.2098 validation: 0.980000\n",
            "Iter-62600 loss: 0.1656 validation: 0.980000\n",
            "Iter-62700 loss: 0.1558 validation: 0.980000\n",
            "Iter-62800 loss: 0.1437 validation: 0.980000\n",
            "Iter-62900 loss: 0.1630 validation: 0.980000\n",
            "Iter-63000 loss: 0.1236 validation: 0.980000\n",
            "Iter-63100 loss: 0.1674 validation: 0.980000\n",
            "Iter-63200 loss: 0.1619 validation: 0.980000\n",
            "Iter-63300 loss: 0.1358 validation: 0.980000\n",
            "Iter-63400 loss: 0.1346 validation: 0.980000\n",
            "Iter-63500 loss: 0.2218 validation: 0.980000\n",
            "Iter-63600 loss: 0.1507 validation: 0.980000\n",
            "Iter-63700 loss: 0.1426 validation: 0.980000\n",
            "Iter-63800 loss: 0.1505 validation: 0.980000\n",
            "Iter-63900 loss: 0.1300 validation: 0.980000\n",
            "Iter-64000 loss: 0.1514 validation: 0.980000\n",
            "Iter-64100 loss: 0.2212 validation: 0.980000\n",
            "Iter-64200 loss: 0.1646 validation: 0.980000\n",
            "Iter-64300 loss: 0.1455 validation: 0.980000\n",
            "Iter-64400 loss: 0.1958 validation: 0.980000\n",
            "Iter-64500 loss: 0.1611 validation: 0.980000\n",
            "Iter-64600 loss: 0.1497 validation: 0.980000\n",
            "Iter-64700 loss: 0.1638 validation: 0.980000\n",
            "Iter-64800 loss: 0.1790 validation: 0.980000\n",
            "Iter-64900 loss: 0.1213 validation: 0.980000\n",
            "Iter-65000 loss: 0.1509 validation: 0.980000\n",
            "Iter-65100 loss: 0.1477 validation: 0.980000\n",
            "Iter-65200 loss: 0.1289 validation: 0.980000\n",
            "Iter-65300 loss: 0.1659 validation: 0.980000\n",
            "Iter-65400 loss: 0.2233 validation: 0.980000\n",
            "Iter-65500 loss: 0.1219 validation: 0.980000\n",
            "Iter-65600 loss: 0.1246 validation: 0.980000\n",
            "Iter-65700 loss: 0.1408 validation: 0.980000\n",
            "Iter-65800 loss: 0.1272 validation: 0.980000\n",
            "Iter-65900 loss: 0.1540 validation: 0.980000\n",
            "Iter-66000 loss: 0.1273 validation: 0.980000\n",
            "Iter-66100 loss: 0.1392 validation: 0.980000\n",
            "Iter-66200 loss: 0.1368 validation: 0.980000\n",
            "Iter-66300 loss: 0.1501 validation: 0.980000\n",
            "Iter-66400 loss: 0.1111 validation: 0.980000\n",
            "Iter-66500 loss: 0.2255 validation: 0.980000\n",
            "Iter-66600 loss: 0.1489 validation: 0.980000\n",
            "Iter-66700 loss: 0.1242 validation: 0.980000\n",
            "Iter-66800 loss: 0.1440 validation: 0.980000\n",
            "Iter-66900 loss: 0.1449 validation: 0.980000\n",
            "Iter-67000 loss: 0.1470 validation: 0.980000\n",
            "Iter-67100 loss: 0.1382 validation: 0.980000\n",
            "Iter-67200 loss: 0.1231 validation: 0.980000\n",
            "Iter-67300 loss: 0.1683 validation: 0.980000\n",
            "Iter-67400 loss: 0.1398 validation: 0.980000\n",
            "Iter-67500 loss: 0.1296 validation: 0.980000\n",
            "Iter-67600 loss: 0.1657 validation: 0.980000\n",
            "Iter-67700 loss: 0.1382 validation: 0.980000\n",
            "Iter-67800 loss: 0.1340 validation: 0.980000\n",
            "Iter-67900 loss: 0.1655 validation: 0.980000\n",
            "Iter-68000 loss: 0.2210 validation: 0.980000\n",
            "Iter-68100 loss: 0.1512 validation: 0.980000\n",
            "Iter-68200 loss: 0.1815 validation: 0.980000\n",
            "Iter-68300 loss: 0.1405 validation: 0.980000\n",
            "Iter-68400 loss: 0.1362 validation: 0.980000\n",
            "Iter-68500 loss: 0.1348 validation: 0.980000\n",
            "Iter-68600 loss: 0.1288 validation: 0.980000\n",
            "Iter-68700 loss: 0.1584 validation: 0.980000\n",
            "Iter-68800 loss: 0.1627 validation: 0.980000\n",
            "Iter-68900 loss: 0.1240 validation: 0.980000\n",
            "Iter-69000 loss: 0.1792 validation: 0.980000\n",
            "Iter-69100 loss: 0.1344 validation: 0.980000\n",
            "Iter-69200 loss: 0.1260 validation: 0.980000\n",
            "Iter-69300 loss: 0.1316 validation: 0.980000\n",
            "Iter-69400 loss: 0.1687 validation: 0.980000\n",
            "Iter-69500 loss: 0.1168 validation: 0.980000\n",
            "Iter-69600 loss: 0.1552 validation: 0.970000\n",
            "Iter-69700 loss: 0.1002 validation: 0.980000\n",
            "Iter-69800 loss: 0.1280 validation: 0.980000\n",
            "Iter-69900 loss: 0.1222 validation: 0.980000\n",
            "Iter-70000 loss: 0.1232 validation: 0.980000\n",
            "Iter-70100 loss: 0.1964 validation: 0.970000\n",
            "Iter-70200 loss: 0.1328 validation: 0.980000\n",
            "Iter-70300 loss: 0.1378 validation: 0.980000\n",
            "Iter-70400 loss: 0.1499 validation: 0.980000\n",
            "Iter-70500 loss: 0.1341 validation: 0.980000\n",
            "Iter-70600 loss: 0.1090 validation: 0.970000\n",
            "Iter-70700 loss: 0.1851 validation: 0.980000\n",
            "Iter-70800 loss: 0.1500 validation: 0.970000\n",
            "Iter-70900 loss: 0.1327 validation: 0.980000\n",
            "Iter-71000 loss: 0.1735 validation: 0.980000\n",
            "Iter-71100 loss: 0.1386 validation: 0.980000\n",
            "Iter-71200 loss: 0.1445 validation: 0.980000\n",
            "Iter-71300 loss: 0.1342 validation: 0.980000\n",
            "Iter-71400 loss: 0.1539 validation: 0.980000\n",
            "Iter-71500 loss: 0.1383 validation: 0.980000\n",
            "Iter-71600 loss: 0.1199 validation: 0.980000\n",
            "Iter-71700 loss: 0.1392 validation: 0.980000\n",
            "Iter-71800 loss: 0.1383 validation: 0.980000\n",
            "Iter-71900 loss: 0.1351 validation: 0.980000\n",
            "Iter-72000 loss: 0.1414 validation: 0.980000\n",
            "Iter-72100 loss: 0.1090 validation: 0.980000\n",
            "Iter-72200 loss: 0.1582 validation: 0.980000\n",
            "Iter-72300 loss: 0.1356 validation: 0.980000\n",
            "Iter-72400 loss: 0.1360 validation: 0.980000\n",
            "Iter-72500 loss: 0.1228 validation: 0.980000\n",
            "Iter-72600 loss: 0.1337 validation: 0.980000\n",
            "Iter-72700 loss: 0.1393 validation: 0.980000\n",
            "Iter-72800 loss: 0.1504 validation: 0.980000\n",
            "Iter-72900 loss: 0.1333 validation: 0.980000\n",
            "Iter-73000 loss: 0.1124 validation: 0.980000\n",
            "Iter-73100 loss: 0.1540 validation: 0.980000\n",
            "Iter-73200 loss: 0.1321 validation: 0.970000\n",
            "Iter-73300 loss: 0.1219 validation: 0.980000\n",
            "Iter-73400 loss: 0.1657 validation: 0.980000\n",
            "Iter-73500 loss: 0.1202 validation: 0.980000\n",
            "Iter-73600 loss: 0.1248 validation: 0.980000\n",
            "Iter-73700 loss: 0.1017 validation: 0.980000\n",
            "Iter-73800 loss: 0.1336 validation: 0.980000\n",
            "Iter-73900 loss: 0.1112 validation: 0.980000\n",
            "Iter-74000 loss: 0.1288 validation: 0.980000\n",
            "Iter-74100 loss: 0.1402 validation: 0.980000\n",
            "Iter-74200 loss: 0.1274 validation: 0.970000\n",
            "Iter-74300 loss: 0.1252 validation: 0.980000\n",
            "Iter-74400 loss: 0.1230 validation: 0.970000\n",
            "Iter-74500 loss: 0.1618 validation: 0.970000\n",
            "Iter-74600 loss: 0.1862 validation: 0.980000\n",
            "Iter-74700 loss: 0.1326 validation: 0.980000\n",
            "Iter-74800 loss: 0.1484 validation: 0.980000\n",
            "Iter-74900 loss: 0.1155 validation: 0.980000\n",
            "Iter-75000 loss: 0.1384 validation: 0.980000\n",
            "Iter-75100 loss: 0.1433 validation: 0.980000\n",
            "Iter-75200 loss: 0.1355 validation: 0.980000\n",
            "Iter-75300 loss: 0.1291 validation: 0.980000\n",
            "Iter-75400 loss: 0.1234 validation: 0.980000\n",
            "Iter-75500 loss: 0.1273 validation: 0.980000\n",
            "Iter-75600 loss: 0.1025 validation: 0.980000\n",
            "Iter-75700 loss: 0.1253 validation: 0.980000\n",
            "Iter-75800 loss: 0.1301 validation: 0.980000\n",
            "Iter-75900 loss: 0.1721 validation: 0.980000\n",
            "Iter-76000 loss: 0.1388 validation: 0.980000\n",
            "Iter-76100 loss: 0.1264 validation: 0.980000\n",
            "Iter-76200 loss: 0.1552 validation: 0.980000\n",
            "Iter-76300 loss: 0.1148 validation: 0.980000\n",
            "Iter-76400 loss: 0.1653 validation: 0.980000\n",
            "Iter-76500 loss: 0.1336 validation: 0.970000\n",
            "Iter-76600 loss: 0.1884 validation: 0.970000\n",
            "Iter-76700 loss: 0.1267 validation: 0.970000\n",
            "Iter-76800 loss: 0.1088 validation: 0.980000\n",
            "Iter-76900 loss: 0.1458 validation: 0.980000\n",
            "Iter-77000 loss: 0.1505 validation: 0.980000\n",
            "Iter-77100 loss: 0.0983 validation: 0.970000\n",
            "Iter-77200 loss: 0.1274 validation: 0.970000\n",
            "Iter-77300 loss: 0.1458 validation: 0.980000\n",
            "Iter-77400 loss: 0.1257 validation: 0.980000\n",
            "Iter-77500 loss: 0.1352 validation: 0.970000\n",
            "Iter-77600 loss: 0.1164 validation: 0.980000\n",
            "Iter-77700 loss: 0.1246 validation: 0.980000\n",
            "Iter-77800 loss: 0.1326 validation: 0.980000\n",
            "Iter-77900 loss: 0.1317 validation: 0.980000\n",
            "Iter-78000 loss: 0.1543 validation: 0.980000\n",
            "Iter-78100 loss: 0.1480 validation: 0.980000\n",
            "Iter-78200 loss: 0.1149 validation: 0.980000\n",
            "Iter-78300 loss: 0.1573 validation: 0.980000\n",
            "Iter-78400 loss: 0.1241 validation: 0.980000\n",
            "Iter-78500 loss: 0.1422 validation: 0.980000\n",
            "Iter-78600 loss: 0.1546 validation: 0.980000\n",
            "Iter-78700 loss: 0.1092 validation: 0.980000\n",
            "Iter-78800 loss: 0.1502 validation: 0.980000\n",
            "Iter-78900 loss: 0.1332 validation: 0.970000\n",
            "Iter-79000 loss: 0.1817 validation: 0.980000\n",
            "Iter-79100 loss: 0.1591 validation: 0.980000\n",
            "Iter-79200 loss: 0.1383 validation: 0.980000\n",
            "Iter-79300 loss: 0.1101 validation: 0.980000\n",
            "Iter-79400 loss: 0.1300 validation: 0.980000\n",
            "Iter-79500 loss: 0.1097 validation: 0.980000\n",
            "Iter-79600 loss: 0.0988 validation: 0.980000\n",
            "Iter-79700 loss: 0.1430 validation: 0.980000\n",
            "Iter-79800 loss: 0.1476 validation: 0.980000\n",
            "Iter-79900 loss: 0.1413 validation: 0.980000\n",
            "Iter-80000 loss: 0.1382 validation: 0.980000\n",
            "Iter-80100 loss: 0.1335 validation: 0.980000\n",
            "Iter-80200 loss: 0.1871 validation: 0.980000\n",
            "Iter-80300 loss: 0.1355 validation: 0.980000\n",
            "Iter-80400 loss: 0.0925 validation: 0.980000\n",
            "Iter-80500 loss: 0.1217 validation: 0.980000\n",
            "Iter-80600 loss: 0.1458 validation: 0.980000\n",
            "Iter-80700 loss: 0.1137 validation: 0.980000\n",
            "Iter-80800 loss: 0.1230 validation: 0.980000\n",
            "Iter-80900 loss: 0.1386 validation: 0.980000\n",
            "Iter-81000 loss: 0.1314 validation: 0.980000\n",
            "Iter-81100 loss: 0.1106 validation: 0.980000\n",
            "Iter-81200 loss: 0.1616 validation: 0.980000\n",
            "Iter-81300 loss: 0.1252 validation: 0.980000\n",
            "Iter-81400 loss: 0.1217 validation: 0.980000\n",
            "Iter-81500 loss: 0.1251 validation: 0.980000\n",
            "Iter-81600 loss: 0.1358 validation: 0.980000\n",
            "Iter-81700 loss: 0.1300 validation: 0.980000\n",
            "Iter-81800 loss: 0.1562 validation: 0.980000\n",
            "Iter-81900 loss: 0.1565 validation: 0.980000\n",
            "Iter-82000 loss: 0.1374 validation: 0.980000\n",
            "Iter-82100 loss: 0.1454 validation: 0.980000\n",
            "Iter-82200 loss: 0.1350 validation: 0.980000\n",
            "Iter-82300 loss: 0.1176 validation: 0.980000\n",
            "Iter-82400 loss: 0.0950 validation: 0.980000\n",
            "Iter-82500 loss: 0.2088 validation: 0.980000\n",
            "Iter-82600 loss: 0.1649 validation: 0.980000\n",
            "Iter-82700 loss: 0.1229 validation: 0.980000\n",
            "Iter-82800 loss: 0.1361 validation: 0.980000\n",
            "Iter-82900 loss: 0.1317 validation: 0.980000\n",
            "Iter-83000 loss: 0.1320 validation: 0.980000\n",
            "Iter-83100 loss: 0.1344 validation: 0.980000\n",
            "Iter-83200 loss: 0.1384 validation: 0.980000\n",
            "Iter-83300 loss: 0.1382 validation: 0.980000\n",
            "Iter-83400 loss: 0.1407 validation: 0.980000\n",
            "Iter-83500 loss: 0.1593 validation: 0.980000\n",
            "Iter-83600 loss: 0.1361 validation: 0.980000\n",
            "Iter-83700 loss: 0.1192 validation: 0.980000\n",
            "Iter-83800 loss: 0.1231 validation: 0.980000\n",
            "Iter-83900 loss: 0.1532 validation: 0.980000\n",
            "Iter-84000 loss: 0.1041 validation: 0.980000\n",
            "Iter-84100 loss: 0.1231 validation: 0.980000\n",
            "Iter-84200 loss: 0.0946 validation: 0.980000\n",
            "Iter-84300 loss: 0.1056 validation: 0.980000\n",
            "Iter-84400 loss: 0.1446 validation: 0.980000\n",
            "Iter-84500 loss: 0.1055 validation: 0.980000\n",
            "Iter-84600 loss: 0.1400 validation: 0.970000\n",
            "Iter-84700 loss: 0.1345 validation: 0.980000\n",
            "Iter-84800 loss: 0.1310 validation: 0.980000\n",
            "Iter-84900 loss: 0.1201 validation: 0.980000\n",
            "Iter-85000 loss: 0.1804 validation: 0.980000\n",
            "Iter-85100 loss: 0.1407 validation: 0.980000\n",
            "Iter-85200 loss: 0.1236 validation: 0.980000\n",
            "Iter-85300 loss: 0.1032 validation: 0.980000\n",
            "Iter-85400 loss: 0.1699 validation: 0.980000\n",
            "Iter-85500 loss: 0.1294 validation: 0.980000\n",
            "Iter-85600 loss: 0.1548 validation: 0.980000\n",
            "Iter-85700 loss: 0.1245 validation: 0.980000\n",
            "Iter-85800 loss: 0.1027 validation: 0.980000\n",
            "Iter-85900 loss: 0.1169 validation: 0.980000\n",
            "Iter-86000 loss: 0.1327 validation: 0.980000\n",
            "Iter-86100 loss: 0.1641 validation: 0.980000\n",
            "Iter-86200 loss: 0.1055 validation: 0.980000\n",
            "Iter-86300 loss: 0.1189 validation: 0.980000\n",
            "Iter-86400 loss: 0.1258 validation: 0.980000\n",
            "Iter-86500 loss: 0.1336 validation: 0.980000\n",
            "Iter-86600 loss: 0.1056 validation: 0.980000\n",
            "Iter-86700 loss: 0.1028 validation: 0.980000\n",
            "Iter-86800 loss: 0.1380 validation: 0.980000\n",
            "Iter-86900 loss: 0.1014 validation: 0.980000\n",
            "Iter-87000 loss: 0.1247 validation: 0.980000\n",
            "Iter-87100 loss: 0.1330 validation: 0.980000\n",
            "Iter-87200 loss: 0.1161 validation: 0.980000\n",
            "Iter-87300 loss: 0.1104 validation: 0.980000\n",
            "Iter-87400 loss: 0.1635 validation: 0.980000\n",
            "Iter-87500 loss: 0.1100 validation: 0.980000\n",
            "Iter-87600 loss: 0.1199 validation: 0.980000\n",
            "Iter-87700 loss: 0.1613 validation: 0.980000\n",
            "Iter-87800 loss: 0.1315 validation: 0.970000\n",
            "Iter-87900 loss: 0.1195 validation: 0.970000\n",
            "Iter-88000 loss: 0.1215 validation: 0.980000\n",
            "Iter-88100 loss: 0.1252 validation: 0.980000\n",
            "Iter-88200 loss: 0.1346 validation: 0.980000\n",
            "Iter-88300 loss: 0.1336 validation: 0.980000\n",
            "Iter-88400 loss: 0.1529 validation: 0.980000\n",
            "Iter-88500 loss: 0.1124 validation: 0.980000\n",
            "Iter-88600 loss: 0.1364 validation: 0.980000\n",
            "Iter-88700 loss: 0.1413 validation: 0.970000\n",
            "Iter-88800 loss: 0.1381 validation: 0.970000\n",
            "Iter-88900 loss: 0.0964 validation: 0.980000\n",
            "Iter-89000 loss: 0.1549 validation: 0.980000\n",
            "Iter-89100 loss: 0.1221 validation: 0.980000\n",
            "Iter-89200 loss: 0.1486 validation: 0.980000\n",
            "Iter-89300 loss: 0.1122 validation: 0.980000\n",
            "Iter-89400 loss: 0.1475 validation: 0.980000\n",
            "Iter-89500 loss: 0.1451 validation: 0.980000\n",
            "Iter-89600 loss: 0.1134 validation: 0.980000\n",
            "Iter-89700 loss: 0.1280 validation: 0.980000\n",
            "Iter-89800 loss: 0.1666 validation: 0.980000\n",
            "Iter-89900 loss: 0.1393 validation: 0.980000\n",
            "Iter-90000 loss: 0.1204 validation: 0.980000\n",
            "Iter-90100 loss: 0.1448 validation: 0.980000\n",
            "Iter-90200 loss: 0.1330 validation: 0.980000\n",
            "Iter-90300 loss: 0.1444 validation: 0.980000\n",
            "Iter-90400 loss: 0.1327 validation: 0.980000\n",
            "Iter-90500 loss: 0.1482 validation: 0.980000\n",
            "Iter-90600 loss: 0.1443 validation: 0.980000\n",
            "Iter-90700 loss: 0.1159 validation: 0.980000\n",
            "Iter-90800 loss: 0.1001 validation: 0.980000\n",
            "Iter-90900 loss: 0.1126 validation: 0.980000\n",
            "Iter-91000 loss: 0.1295 validation: 0.980000\n",
            "Iter-91100 loss: 0.1211 validation: 0.980000\n",
            "Iter-91200 loss: 0.1360 validation: 0.980000\n",
            "Iter-91300 loss: 0.1290 validation: 0.980000\n",
            "Iter-91400 loss: 0.1477 validation: 0.980000\n",
            "Iter-91500 loss: 0.1426 validation: 0.970000\n",
            "Iter-91600 loss: 0.1212 validation: 0.980000\n",
            "Iter-91700 loss: 0.1111 validation: 0.980000\n",
            "Iter-91800 loss: 0.1295 validation: 0.980000\n",
            "Iter-91900 loss: 0.1386 validation: 0.980000\n",
            "Iter-92000 loss: 0.1377 validation: 0.980000\n",
            "Iter-92100 loss: 0.1203 validation: 0.980000\n",
            "Iter-92200 loss: 0.1228 validation: 0.980000\n",
            "Iter-92300 loss: 0.1303 validation: 0.980000\n",
            "Iter-92400 loss: 0.1005 validation: 0.980000\n",
            "Iter-92500 loss: 0.1188 validation: 0.980000\n",
            "Iter-92600 loss: 0.1017 validation: 0.980000\n",
            "Iter-92700 loss: 0.1420 validation: 0.970000\n",
            "Iter-92800 loss: 0.1290 validation: 0.980000\n",
            "Iter-92900 loss: 0.1591 validation: 0.980000\n",
            "Iter-93000 loss: 0.1081 validation: 0.980000\n",
            "Iter-93100 loss: 0.1067 validation: 0.980000\n",
            "Iter-93200 loss: 0.1100 validation: 0.980000\n",
            "Iter-93300 loss: 0.1207 validation: 0.980000\n",
            "Iter-93400 loss: 0.1161 validation: 0.980000\n",
            "Iter-93500 loss: 0.1433 validation: 0.980000\n",
            "Iter-93600 loss: 0.1487 validation: 0.980000\n",
            "Iter-93700 loss: 0.1004 validation: 0.980000\n",
            "Iter-93800 loss: 0.1072 validation: 0.980000\n",
            "Iter-93900 loss: 0.1144 validation: 0.980000\n",
            "Iter-94000 loss: 0.1218 validation: 0.980000\n",
            "Iter-94100 loss: 0.1184 validation: 0.980000\n",
            "Iter-94200 loss: 0.1084 validation: 0.970000\n",
            "Iter-94300 loss: 0.1120 validation: 0.980000\n",
            "Iter-94400 loss: 0.1028 validation: 0.980000\n",
            "Iter-94500 loss: 0.1192 validation: 0.980000\n",
            "Iter-94600 loss: 0.1147 validation: 0.980000\n",
            "Iter-94700 loss: 0.1048 validation: 0.980000\n",
            "Iter-94800 loss: 0.1390 validation: 0.980000\n",
            "Iter-94900 loss: 0.1404 validation: 0.980000\n",
            "Iter-95000 loss: 0.1056 validation: 0.970000\n",
            "Iter-95100 loss: 0.1092 validation: 0.980000\n",
            "Iter-95200 loss: 0.1154 validation: 0.980000\n",
            "Iter-95300 loss: 0.1211 validation: 0.980000\n",
            "Iter-95400 loss: 0.1363 validation: 0.980000\n",
            "Iter-95500 loss: 0.1311 validation: 0.980000\n",
            "Iter-95600 loss: 0.1307 validation: 0.980000\n",
            "Iter-95700 loss: 0.1458 validation: 0.980000\n",
            "Iter-95800 loss: 0.1249 validation: 0.980000\n",
            "Iter-95900 loss: 0.1139 validation: 0.980000\n",
            "Iter-96000 loss: 0.1505 validation: 0.970000\n",
            "Iter-96100 loss: 0.1070 validation: 0.970000\n",
            "Iter-96200 loss: 0.1273 validation: 0.970000\n",
            "Iter-96300 loss: 0.1133 validation: 0.980000\n",
            "Iter-96400 loss: 0.1327 validation: 0.980000\n",
            "Iter-96500 loss: 0.1292 validation: 0.980000\n",
            "Iter-96600 loss: 0.1131 validation: 0.980000\n",
            "Iter-96700 loss: 0.1171 validation: 0.980000\n",
            "Iter-96800 loss: 0.1388 validation: 0.980000\n",
            "Iter-96900 loss: 0.1119 validation: 0.970000\n",
            "Iter-97000 loss: 0.1320 validation: 0.980000\n",
            "Iter-97100 loss: 0.1133 validation: 0.980000\n",
            "Iter-97200 loss: 0.0997 validation: 0.980000\n",
            "Iter-97300 loss: 0.1277 validation: 0.980000\n",
            "Iter-97400 loss: 0.1100 validation: 0.980000\n",
            "Iter-97500 loss: 0.0922 validation: 0.980000\n",
            "Iter-97600 loss: 0.1340 validation: 0.980000\n",
            "Iter-97700 loss: 0.1547 validation: 0.970000\n",
            "Iter-97800 loss: 0.1683 validation: 0.980000\n",
            "Iter-97900 loss: 0.1356 validation: 0.980000\n",
            "Iter-98000 loss: 0.1514 validation: 0.980000\n",
            "Iter-98100 loss: 0.1122 validation: 0.980000\n",
            "Iter-98200 loss: 0.1169 validation: 0.980000\n",
            "Iter-98300 loss: 0.1048 validation: 0.980000\n",
            "Iter-98400 loss: 0.1050 validation: 0.980000\n",
            "Iter-98500 loss: 0.1266 validation: 0.980000\n",
            "Iter-98600 loss: 0.1241 validation: 0.980000\n",
            "Iter-98700 loss: 0.1051 validation: 0.980000\n",
            "Iter-98800 loss: 0.1255 validation: 0.980000\n",
            "Iter-98900 loss: 0.1151 validation: 0.980000\n",
            "Iter-99000 loss: 0.1208 validation: 0.980000\n",
            "Iter-99100 loss: 0.1199 validation: 0.980000\n",
            "Iter-99200 loss: 0.1377 validation: 0.980000\n",
            "Iter-99300 loss: 0.1201 validation: 0.980000\n",
            "Iter-99400 loss: 0.1055 validation: 0.980000\n",
            "Iter-99500 loss: 0.1041 validation: 0.980000\n",
            "Iter-99600 loss: 0.1204 validation: 0.980000\n",
            "Iter-99700 loss: 0.1241 validation: 0.980000\n",
            "Iter-99800 loss: 0.1369 validation: 0.980000\n",
            "Iter-99900 loss: 0.1338 validation: 0.970000\n",
            "Iter-100000 loss: 0.0989 validation: 0.980000\n",
            "\n",
            "Best max_norm:  4.0 best dropout rate for input:  0.85 best dropout rate for hidden layers:  0.5 and result accuracy is:  0.9823\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "N_8rF0QG36Ck"
      },
      "source": [
        "Dropout NN + max-norm constraint + ReLU + 3 layers + 1024 hidden units (50,000 weight updates)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wPM6pQCYlBNl",
        "outputId": "54a1209e-57cc-4e8c-a9d4-a5bf556adb57"
      },
      "source": [
        "model = 'ff'\n",
        "n_iter = 50000\n",
        "alpha = 1e-3\n",
        "mb_size = 64\n",
        "n_experiment = 1\n",
        "reg = 1e-5\n",
        "print_after = 100\n",
        "p_dropout = [0.8,0.5] #if nlayer=n, len(p_dropout) = n-1\n",
        "loss = 'cross_ent'\n",
        "nonlin = 'relu'\n",
        "solver = 'sgd'\n",
        "max_norm = 4\n",
        "nlayer = 3\n",
        "\n",
        "# Define default candidates\n",
        "candidates = np.array([[3, 3.5, 4],[0.795, 0.8, 0.85],[0.495, 0.5, 0.55]]) # first hyper-param as max-norm, second as drop-out rate for input layer, third as drop-out rate for hidden layers\n",
        "\n",
        "run_mnist(1024, model, n_iter, alpha, mb_size, n_experiment, reg, print_after, p_dropout, loss, nonlin, solver, max_norm, x_train, x_validation, x_test, y_train, y_validation, y_test, candidates, nlayer)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Experimenting on sgd\n",
            "\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.5913 validation: 0.800000\n",
            "Iter-200 loss: 1.3298 validation: 0.820000\n",
            "Iter-300 loss: 1.1730 validation: 0.870000\n",
            "Iter-400 loss: 1.0746 validation: 0.890000\n",
            "Iter-500 loss: 1.0618 validation: 0.910000\n",
            "Iter-600 loss: 0.9404 validation: 0.920000\n",
            "Iter-700 loss: 0.9870 validation: 0.920000\n",
            "Iter-800 loss: 1.0305 validation: 0.920000\n",
            "Iter-900 loss: 0.9081 validation: 0.930000\n",
            "Iter-1000 loss: 0.8818 validation: 0.950000\n",
            "Iter-1100 loss: 0.9306 validation: 0.940000\n",
            "Iter-1200 loss: 0.7511 validation: 0.940000\n",
            "Iter-1300 loss: 0.7967 validation: 0.950000\n",
            "Iter-1400 loss: 0.7527 validation: 0.950000\n",
            "Iter-1500 loss: 0.8118 validation: 0.950000\n",
            "Iter-1600 loss: 0.7876 validation: 0.950000\n",
            "Iter-1700 loss: 0.7678 validation: 0.950000\n",
            "Iter-1800 loss: 0.7153 validation: 0.950000\n",
            "Iter-1900 loss: 0.7632 validation: 0.950000\n",
            "Iter-2000 loss: 0.7702 validation: 0.950000\n",
            "Iter-2100 loss: 0.6716 validation: 0.950000\n",
            "Iter-2200 loss: 0.6313 validation: 0.950000\n",
            "Iter-2300 loss: 0.6792 validation: 0.950000\n",
            "Iter-2400 loss: 0.7033 validation: 0.950000\n",
            "Iter-2500 loss: 0.6877 validation: 0.950000\n",
            "Iter-2600 loss: 0.6341 validation: 0.950000\n",
            "Iter-2700 loss: 0.6922 validation: 0.950000\n",
            "Iter-2800 loss: 0.5545 validation: 0.950000\n",
            "Iter-2900 loss: 0.6561 validation: 0.950000\n",
            "Iter-3000 loss: 0.5919 validation: 0.950000\n",
            "Iter-3100 loss: 0.6481 validation: 0.950000\n",
            "Iter-3200 loss: 0.6767 validation: 0.950000\n",
            "Iter-3300 loss: 0.5608 validation: 0.950000\n",
            "Iter-3400 loss: 0.6517 validation: 0.950000\n",
            "Iter-3500 loss: 0.6156 validation: 0.950000\n",
            "Iter-3600 loss: 0.5862 validation: 0.950000\n",
            "Iter-3700 loss: 0.7224 validation: 0.950000\n",
            "Iter-3800 loss: 0.5448 validation: 0.950000\n",
            "Iter-3900 loss: 0.5944 validation: 0.950000\n",
            "Iter-4000 loss: 0.6091 validation: 0.950000\n",
            "Iter-4100 loss: 0.5663 validation: 0.950000\n",
            "Iter-4200 loss: 0.5874 validation: 0.950000\n",
            "Iter-4300 loss: 0.5758 validation: 0.950000\n",
            "Iter-4400 loss: 0.4977 validation: 0.950000\n",
            "Iter-4500 loss: 0.5063 validation: 0.950000\n",
            "Iter-4600 loss: 0.5081 validation: 0.960000\n",
            "Iter-4700 loss: 0.5800 validation: 0.960000\n",
            "Iter-4800 loss: 0.6502 validation: 0.950000\n",
            "Iter-4900 loss: 0.5961 validation: 0.950000\n",
            "Iter-5000 loss: 0.5151 validation: 0.960000\n",
            "Iter-5100 loss: 0.5058 validation: 0.950000\n",
            "Iter-5200 loss: 0.5022 validation: 0.950000\n",
            "Iter-5300 loss: 0.5625 validation: 0.950000\n",
            "Iter-5400 loss: 0.4641 validation: 0.960000\n",
            "Iter-5500 loss: 0.4992 validation: 0.950000\n",
            "Iter-5600 loss: 0.4185 validation: 0.970000\n",
            "Iter-5700 loss: 0.4839 validation: 0.960000\n",
            "Iter-5800 loss: 0.5152 validation: 0.970000\n",
            "Iter-5900 loss: 0.5320 validation: 0.960000\n",
            "Iter-6000 loss: 0.4656 validation: 0.970000\n",
            "Iter-6100 loss: 0.5680 validation: 0.960000\n",
            "Iter-6200 loss: 0.5118 validation: 0.960000\n",
            "Iter-6300 loss: 0.3871 validation: 0.970000\n",
            "Iter-6400 loss: 0.4643 validation: 0.960000\n",
            "Iter-6500 loss: 0.5037 validation: 0.960000\n",
            "Iter-6600 loss: 0.3972 validation: 0.970000\n",
            "Iter-6700 loss: 0.4449 validation: 0.970000\n",
            "Iter-6800 loss: 0.4498 validation: 0.970000\n",
            "Iter-6900 loss: 0.4985 validation: 0.970000\n",
            "Iter-7000 loss: 0.5147 validation: 0.970000\n",
            "Iter-7100 loss: 0.4414 validation: 0.970000\n",
            "Iter-7200 loss: 0.3944 validation: 0.970000\n",
            "Iter-7300 loss: 0.5400 validation: 0.970000\n",
            "Iter-7400 loss: 0.3675 validation: 0.970000\n",
            "Iter-7500 loss: 0.4885 validation: 0.970000\n",
            "Iter-7600 loss: 0.5506 validation: 0.970000\n",
            "Iter-7700 loss: 0.4307 validation: 0.970000\n",
            "Iter-7800 loss: 0.5502 validation: 0.970000\n",
            "Iter-7900 loss: 0.4264 validation: 0.970000\n",
            "Iter-8000 loss: 0.3912 validation: 0.970000\n",
            "Iter-8100 loss: 0.3571 validation: 0.970000\n",
            "Iter-8200 loss: 0.3811 validation: 0.970000\n",
            "Iter-8300 loss: 0.4892 validation: 0.970000\n",
            "Iter-8400 loss: 0.3552 validation: 0.970000\n",
            "Iter-8500 loss: 0.4942 validation: 0.970000\n",
            "Iter-8600 loss: 0.3666 validation: 0.970000\n",
            "Iter-8700 loss: 0.4285 validation: 0.970000\n",
            "Iter-8800 loss: 0.3790 validation: 0.970000\n",
            "Iter-8900 loss: 0.3771 validation: 0.970000\n",
            "Iter-9000 loss: 0.5232 validation: 0.970000\n",
            "Iter-9100 loss: 0.4373 validation: 0.970000\n",
            "Iter-9200 loss: 0.4200 validation: 0.970000\n",
            "Iter-9300 loss: 0.4503 validation: 0.970000\n",
            "Iter-9400 loss: 0.3987 validation: 0.970000\n",
            "Iter-9500 loss: 0.2963 validation: 0.970000\n",
            "Iter-9600 loss: 0.3749 validation: 0.970000\n",
            "Iter-9700 loss: 0.4722 validation: 0.970000\n",
            "Iter-9800 loss: 0.3808 validation: 0.970000\n",
            "Iter-9900 loss: 0.4131 validation: 0.970000\n",
            "Iter-10000 loss: 0.3109 validation: 0.970000\n",
            "Iter-10100 loss: 0.4077 validation: 0.970000\n",
            "Iter-10200 loss: 0.3332 validation: 0.970000\n",
            "Iter-10300 loss: 0.3939 validation: 0.970000\n",
            "Iter-10400 loss: 0.3756 validation: 0.970000\n",
            "Iter-10500 loss: 0.3479 validation: 0.970000\n",
            "Iter-10600 loss: 0.3583 validation: 0.970000\n",
            "Iter-10700 loss: 0.2804 validation: 0.970000\n",
            "Iter-10800 loss: 0.3791 validation: 0.970000\n",
            "Iter-10900 loss: 0.4096 validation: 0.970000\n",
            "Iter-11000 loss: 0.4449 validation: 0.970000\n",
            "Iter-11100 loss: 0.3888 validation: 0.970000\n",
            "Iter-11200 loss: 0.3064 validation: 0.970000\n",
            "Iter-11300 loss: 0.4028 validation: 0.970000\n",
            "Iter-11400 loss: 0.2832 validation: 0.970000\n",
            "Iter-11500 loss: 0.3149 validation: 0.970000\n",
            "Iter-11600 loss: 0.3313 validation: 0.970000\n",
            "Iter-11700 loss: 0.3750 validation: 0.970000\n",
            "Iter-11800 loss: 0.3702 validation: 0.970000\n",
            "Iter-11900 loss: 0.3024 validation: 0.970000\n",
            "Iter-12000 loss: 0.3623 validation: 0.970000\n",
            "Iter-12100 loss: 0.3032 validation: 0.970000\n",
            "Iter-12200 loss: 0.2816 validation: 0.970000\n",
            "Iter-12300 loss: 0.3531 validation: 0.970000\n",
            "Iter-12400 loss: 0.3372 validation: 0.970000\n",
            "Iter-12500 loss: 0.3109 validation: 0.970000\n",
            "Iter-12600 loss: 0.3356 validation: 0.970000\n",
            "Iter-12700 loss: 0.3668 validation: 0.970000\n",
            "Iter-12800 loss: 0.3133 validation: 0.970000\n",
            "Iter-12900 loss: 0.3178 validation: 0.970000\n",
            "Iter-13000 loss: 0.4293 validation: 0.970000\n",
            "Iter-13100 loss: 0.2639 validation: 0.970000\n",
            "Iter-13200 loss: 0.3562 validation: 0.970000\n",
            "Iter-13300 loss: 0.2711 validation: 0.970000\n",
            "Iter-13400 loss: 0.2636 validation: 0.970000\n",
            "Iter-13500 loss: 0.3949 validation: 0.970000\n",
            "Iter-13600 loss: 0.3446 validation: 0.970000\n",
            "Iter-13700 loss: 0.4053 validation: 0.970000\n",
            "Iter-13800 loss: 0.2992 validation: 0.970000\n",
            "Iter-13900 loss: 0.2452 validation: 0.970000\n",
            "Iter-14000 loss: 0.4025 validation: 0.970000\n",
            "Iter-14100 loss: 0.2199 validation: 0.970000\n",
            "Iter-14200 loss: 0.3065 validation: 0.970000\n",
            "Iter-14300 loss: 0.2863 validation: 0.970000\n",
            "Iter-14400 loss: 0.3314 validation: 0.970000\n",
            "Iter-14500 loss: 0.3635 validation: 0.970000\n",
            "Iter-14600 loss: 0.3180 validation: 0.970000\n",
            "Iter-14700 loss: 0.2601 validation: 0.970000\n",
            "Iter-14800 loss: 0.3066 validation: 0.970000\n",
            "Iter-14900 loss: 0.3079 validation: 0.970000\n",
            "Iter-15000 loss: 0.3146 validation: 0.970000\n",
            "Iter-15100 loss: 0.3069 validation: 0.970000\n",
            "Iter-15200 loss: 0.2641 validation: 0.970000\n",
            "Iter-15300 loss: 0.3124 validation: 0.970000\n",
            "Iter-15400 loss: 0.3770 validation: 0.970000\n",
            "Iter-15500 loss: 0.3176 validation: 0.970000\n",
            "Iter-15600 loss: 0.3503 validation: 0.970000\n",
            "Iter-15700 loss: 0.2726 validation: 0.970000\n",
            "Iter-15800 loss: 0.2794 validation: 0.970000\n",
            "Iter-15900 loss: 0.3404 validation: 0.970000\n",
            "Iter-16000 loss: 0.3812 validation: 0.970000\n",
            "Iter-16100 loss: 0.2457 validation: 0.970000\n",
            "Iter-16200 loss: 0.2658 validation: 0.970000\n",
            "Iter-16300 loss: 0.3276 validation: 0.970000\n",
            "Iter-16400 loss: 0.2851 validation: 0.970000\n",
            "Iter-16500 loss: 0.2738 validation: 0.970000\n",
            "Iter-16600 loss: 0.2879 validation: 0.970000\n",
            "Iter-16700 loss: 0.3573 validation: 0.970000\n",
            "Iter-16800 loss: 0.2307 validation: 0.970000\n",
            "Iter-16900 loss: 0.3177 validation: 0.970000\n",
            "Iter-17000 loss: 0.2506 validation: 0.970000\n",
            "Iter-17100 loss: 0.2723 validation: 0.970000\n",
            "Iter-17200 loss: 0.2921 validation: 0.970000\n",
            "Iter-17300 loss: 0.3030 validation: 0.970000\n",
            "Iter-17400 loss: 0.2588 validation: 0.970000\n",
            "Iter-17500 loss: 0.3182 validation: 0.970000\n",
            "Iter-17600 loss: 0.2689 validation: 0.970000\n",
            "Iter-17700 loss: 0.2648 validation: 0.970000\n",
            "Iter-17800 loss: 0.2866 validation: 0.970000\n",
            "Iter-17900 loss: 0.2634 validation: 0.970000\n",
            "Iter-18000 loss: 0.2261 validation: 0.970000\n",
            "Iter-18100 loss: 0.2476 validation: 0.970000\n",
            "Iter-18200 loss: 0.2629 validation: 0.970000\n",
            "Iter-18300 loss: 0.2774 validation: 0.970000\n",
            "Iter-18400 loss: 0.2752 validation: 0.970000\n",
            "Iter-18500 loss: 0.2300 validation: 0.970000\n",
            "Iter-18600 loss: 0.2679 validation: 0.970000\n",
            "Iter-18700 loss: 0.3565 validation: 0.970000\n",
            "Iter-18800 loss: 0.2425 validation: 0.970000\n",
            "Iter-18900 loss: 0.2595 validation: 0.970000\n",
            "Iter-19000 loss: 0.2239 validation: 0.970000\n",
            "Iter-19100 loss: 0.2455 validation: 0.970000\n",
            "Iter-19200 loss: 0.2487 validation: 0.970000\n",
            "Iter-19300 loss: 0.2959 validation: 0.970000\n",
            "Iter-19400 loss: 0.3206 validation: 0.970000\n",
            "Iter-19500 loss: 0.2427 validation: 0.970000\n",
            "Iter-19600 loss: 0.3404 validation: 0.970000\n",
            "Iter-19700 loss: 0.2626 validation: 0.970000\n",
            "Iter-19800 loss: 0.1959 validation: 0.970000\n",
            "Iter-19900 loss: 0.2830 validation: 0.970000\n",
            "Iter-20000 loss: 0.3427 validation: 0.970000\n",
            "Iter-20100 loss: 0.2399 validation: 0.970000\n",
            "Iter-20200 loss: 0.2757 validation: 0.970000\n",
            "Iter-20300 loss: 0.2742 validation: 0.970000\n",
            "Iter-20400 loss: 0.2428 validation: 0.970000\n",
            "Iter-20500 loss: 0.2609 validation: 0.970000\n",
            "Iter-20600 loss: 0.2467 validation: 0.970000\n",
            "Iter-20700 loss: 0.2987 validation: 0.970000\n",
            "Iter-20800 loss: 0.2874 validation: 0.970000\n",
            "Iter-20900 loss: 0.2714 validation: 0.970000\n",
            "Iter-21000 loss: 0.2755 validation: 0.970000\n",
            "Iter-21100 loss: 0.2726 validation: 0.970000\n",
            "Iter-21200 loss: 0.2950 validation: 0.970000\n",
            "Iter-21300 loss: 0.4190 validation: 0.970000\n",
            "Iter-21400 loss: 0.2738 validation: 0.970000\n",
            "Iter-21500 loss: 0.2190 validation: 0.970000\n",
            "Iter-21600 loss: 0.2592 validation: 0.970000\n",
            "Iter-21700 loss: 0.3041 validation: 0.970000\n",
            "Iter-21800 loss: 0.2716 validation: 0.970000\n",
            "Iter-21900 loss: 0.1822 validation: 0.970000\n",
            "Iter-22000 loss: 0.2353 validation: 0.970000\n",
            "Iter-22100 loss: 0.2972 validation: 0.970000\n",
            "Iter-22200 loss: 0.3024 validation: 0.970000\n",
            "Iter-22300 loss: 0.3130 validation: 0.970000\n",
            "Iter-22400 loss: 0.2159 validation: 0.970000\n",
            "Iter-22500 loss: 0.2166 validation: 0.970000\n",
            "Iter-22600 loss: 0.1859 validation: 0.970000\n",
            "Iter-22700 loss: 0.2121 validation: 0.970000\n",
            "Iter-22800 loss: 0.1923 validation: 0.970000\n",
            "Iter-22900 loss: 0.2416 validation: 0.970000\n",
            "Iter-23000 loss: 0.2666 validation: 0.970000\n",
            "Iter-23100 loss: 0.2655 validation: 0.970000\n",
            "Iter-23200 loss: 0.2995 validation: 0.970000\n",
            "Iter-23300 loss: 0.2151 validation: 0.970000\n",
            "Iter-23400 loss: 0.3084 validation: 0.970000\n",
            "Iter-23500 loss: 0.2098 validation: 0.970000\n",
            "Iter-23600 loss: 0.2655 validation: 0.970000\n",
            "Iter-23700 loss: 0.2788 validation: 0.970000\n",
            "Iter-23800 loss: 0.2934 validation: 0.970000\n",
            "Iter-23900 loss: 0.2473 validation: 0.970000\n",
            "Iter-24000 loss: 0.2574 validation: 0.970000\n",
            "Iter-24100 loss: 0.2408 validation: 0.970000\n",
            "Iter-24200 loss: 0.2535 validation: 0.970000\n",
            "Iter-24300 loss: 0.2942 validation: 0.970000\n",
            "Iter-24400 loss: 0.3340 validation: 0.970000\n",
            "Iter-24500 loss: 0.2750 validation: 0.970000\n",
            "Iter-24600 loss: 0.2276 validation: 0.970000\n",
            "Iter-24700 loss: 0.2016 validation: 0.970000\n",
            "Iter-24800 loss: 0.3556 validation: 0.970000\n",
            "Iter-24900 loss: 0.2129 validation: 0.970000\n",
            "Iter-25000 loss: 0.2172 validation: 0.970000\n",
            "Iter-25100 loss: 0.2608 validation: 0.970000\n",
            "Iter-25200 loss: 0.2020 validation: 0.970000\n",
            "Iter-25300 loss: 0.2507 validation: 0.970000\n",
            "Iter-25400 loss: 0.1994 validation: 0.970000\n",
            "Iter-25500 loss: 0.2220 validation: 0.970000\n",
            "Iter-25600 loss: 0.2156 validation: 0.970000\n",
            "Iter-25700 loss: 0.2088 validation: 0.970000\n",
            "Iter-25800 loss: 0.2701 validation: 0.970000\n",
            "Iter-25900 loss: 0.2401 validation: 0.970000\n",
            "Iter-26000 loss: 0.2246 validation: 0.970000\n",
            "Iter-26100 loss: 0.2479 validation: 0.970000\n",
            "Iter-26200 loss: 0.3077 validation: 0.970000\n",
            "Iter-26300 loss: 0.2605 validation: 0.970000\n",
            "Iter-26400 loss: 0.2061 validation: 0.970000\n",
            "Iter-26500 loss: 0.2723 validation: 0.970000\n",
            "Iter-26600 loss: 0.2169 validation: 0.970000\n",
            "Iter-26700 loss: 0.3365 validation: 0.970000\n",
            "Iter-26800 loss: 0.2384 validation: 0.970000\n",
            "Iter-26900 loss: 0.1834 validation: 0.970000\n",
            "Iter-27000 loss: 0.2139 validation: 0.970000\n",
            "Iter-27100 loss: 0.2587 validation: 0.970000\n",
            "Iter-27200 loss: 0.1975 validation: 0.970000\n",
            "Iter-27300 loss: 0.2027 validation: 0.970000\n",
            "Iter-27400 loss: 0.2487 validation: 0.970000\n",
            "Iter-27500 loss: 0.2218 validation: 0.970000\n",
            "Iter-27600 loss: 0.2131 validation: 0.970000\n",
            "Iter-27700 loss: 0.1927 validation: 0.970000\n",
            "Iter-27800 loss: 0.2762 validation: 0.970000\n",
            "Iter-27900 loss: 0.2289 validation: 0.970000\n",
            "Iter-28000 loss: 0.2536 validation: 0.970000\n",
            "Iter-28100 loss: 0.1873 validation: 0.970000\n",
            "Iter-28200 loss: 0.3019 validation: 0.970000\n",
            "Iter-28300 loss: 0.1807 validation: 0.970000\n",
            "Iter-28400 loss: 0.2049 validation: 0.970000\n",
            "Iter-28500 loss: 0.2507 validation: 0.970000\n",
            "Iter-28600 loss: 0.1894 validation: 0.970000\n",
            "Iter-28700 loss: 0.2034 validation: 0.970000\n",
            "Iter-28800 loss: 0.2005 validation: 0.970000\n",
            "Iter-28900 loss: 0.2384 validation: 0.970000\n",
            "Iter-29000 loss: 0.2553 validation: 0.970000\n",
            "Iter-29100 loss: 0.2128 validation: 0.970000\n",
            "Iter-29200 loss: 0.2159 validation: 0.970000\n",
            "Iter-29300 loss: 0.1751 validation: 0.970000\n",
            "Iter-29400 loss: 0.2137 validation: 0.970000\n",
            "Iter-29500 loss: 0.1938 validation: 0.970000\n",
            "Iter-29600 loss: 0.2278 validation: 0.970000\n",
            "Iter-29700 loss: 0.1822 validation: 0.970000\n",
            "Iter-29800 loss: 0.2209 validation: 0.970000\n",
            "Iter-29900 loss: 0.2310 validation: 0.970000\n",
            "Iter-30000 loss: 0.3138 validation: 0.970000\n",
            "Iter-30100 loss: 0.2709 validation: 0.970000\n",
            "Iter-30200 loss: 0.2060 validation: 0.970000\n",
            "Iter-30300 loss: 0.2549 validation: 0.980000\n",
            "Iter-30400 loss: 0.1805 validation: 0.970000\n",
            "Iter-30500 loss: 0.3297 validation: 0.970000\n",
            "Iter-30600 loss: 0.2669 validation: 0.970000\n",
            "Iter-30700 loss: 0.2148 validation: 0.970000\n",
            "Iter-30800 loss: 0.1561 validation: 0.970000\n",
            "Iter-30900 loss: 0.2061 validation: 0.970000\n",
            "Iter-31000 loss: 0.2121 validation: 0.970000\n",
            "Iter-31100 loss: 0.2043 validation: 0.970000\n",
            "Iter-31200 loss: 0.2349 validation: 0.970000\n",
            "Iter-31300 loss: 0.2474 validation: 0.970000\n",
            "Iter-31400 loss: 0.1624 validation: 0.970000\n",
            "Iter-31500 loss: 0.2002 validation: 0.970000\n",
            "Iter-31600 loss: 0.1929 validation: 0.970000\n",
            "Iter-31700 loss: 0.2114 validation: 0.970000\n",
            "Iter-31800 loss: 0.1588 validation: 0.970000\n",
            "Iter-31900 loss: 0.1838 validation: 0.970000\n",
            "Iter-32000 loss: 0.1951 validation: 0.970000\n",
            "Iter-32100 loss: 0.2193 validation: 0.970000\n",
            "Iter-32200 loss: 0.2034 validation: 0.970000\n",
            "Iter-32300 loss: 0.1591 validation: 0.970000\n",
            "Iter-32400 loss: 0.1810 validation: 0.970000\n",
            "Iter-32500 loss: 0.2369 validation: 0.970000\n",
            "Iter-32600 loss: 0.2264 validation: 0.970000\n",
            "Iter-32700 loss: 0.2037 validation: 0.970000\n",
            "Iter-32800 loss: 0.2293 validation: 0.970000\n",
            "Iter-32900 loss: 0.1977 validation: 0.970000\n",
            "Iter-33000 loss: 0.1679 validation: 0.970000\n",
            "Iter-33100 loss: 0.2421 validation: 0.970000\n",
            "Iter-33200 loss: 0.2127 validation: 0.970000\n",
            "Iter-33300 loss: 0.1905 validation: 0.970000\n",
            "Iter-33400 loss: 0.2101 validation: 0.970000\n",
            "Iter-33500 loss: 0.2466 validation: 0.970000\n",
            "Iter-33600 loss: 0.1943 validation: 0.970000\n",
            "Iter-33700 loss: 0.1637 validation: 0.970000\n",
            "Iter-33800 loss: 0.2015 validation: 0.970000\n",
            "Iter-33900 loss: 0.1886 validation: 0.970000\n",
            "Iter-34000 loss: 0.2566 validation: 0.970000\n",
            "Iter-34100 loss: 0.2548 validation: 0.970000\n",
            "Iter-34200 loss: 0.1832 validation: 0.970000\n",
            "Iter-34300 loss: 0.1631 validation: 0.970000\n",
            "Iter-34400 loss: 0.1594 validation: 0.970000\n",
            "Iter-34500 loss: 0.2222 validation: 0.970000\n",
            "Iter-34600 loss: 0.1910 validation: 0.970000\n",
            "Iter-34700 loss: 0.1856 validation: 0.970000\n",
            "Iter-34800 loss: 0.2541 validation: 0.970000\n",
            "Iter-34900 loss: 0.2352 validation: 0.970000\n",
            "Iter-35000 loss: 0.2241 validation: 0.970000\n",
            "Iter-35100 loss: 0.1733 validation: 0.970000\n",
            "Iter-35200 loss: 0.2347 validation: 0.970000\n",
            "Iter-35300 loss: 0.1623 validation: 0.970000\n",
            "Iter-35400 loss: 0.1815 validation: 0.970000\n",
            "Iter-35500 loss: 0.1841 validation: 0.970000\n",
            "Iter-35600 loss: 0.1727 validation: 0.970000\n",
            "Iter-35700 loss: 0.1954 validation: 0.970000\n",
            "Iter-35800 loss: 0.2164 validation: 0.970000\n",
            "Iter-35900 loss: 0.1928 validation: 0.970000\n",
            "Iter-36000 loss: 0.1943 validation: 0.970000\n",
            "Iter-36100 loss: 0.1796 validation: 0.970000\n",
            "Iter-36200 loss: 0.1768 validation: 0.970000\n",
            "Iter-36300 loss: 0.1856 validation: 0.970000\n",
            "Iter-36400 loss: 0.1504 validation: 0.970000\n",
            "Iter-36500 loss: 0.2082 validation: 0.970000\n",
            "Iter-36600 loss: 0.1749 validation: 0.970000\n",
            "Iter-36700 loss: 0.2056 validation: 0.970000\n",
            "Iter-36800 loss: 0.2420 validation: 0.970000\n",
            "Iter-36900 loss: 0.1878 validation: 0.970000\n",
            "Iter-37000 loss: 0.1799 validation: 0.970000\n",
            "Iter-37100 loss: 0.1649 validation: 0.970000\n",
            "Iter-37200 loss: 0.1669 validation: 0.970000\n",
            "Iter-37300 loss: 0.2248 validation: 0.970000\n",
            "Iter-37400 loss: 0.1630 validation: 0.970000\n",
            "Iter-37500 loss: 0.1942 validation: 0.970000\n",
            "Iter-37600 loss: 0.1854 validation: 0.970000\n",
            "Iter-37700 loss: 0.1681 validation: 0.970000\n",
            "Iter-37800 loss: 0.2518 validation: 0.970000\n",
            "Iter-37900 loss: 0.1846 validation: 0.970000\n",
            "Iter-38000 loss: 0.1794 validation: 0.970000\n",
            "Iter-38100 loss: 0.2053 validation: 0.970000\n",
            "Iter-38200 loss: 0.2152 validation: 0.970000\n",
            "Iter-38300 loss: 0.2094 validation: 0.970000\n",
            "Iter-38400 loss: 0.1844 validation: 0.970000\n",
            "Iter-38500 loss: 0.1733 validation: 0.970000\n",
            "Iter-38600 loss: 0.2079 validation: 0.970000\n",
            "Iter-38700 loss: 0.2258 validation: 0.970000\n",
            "Iter-38800 loss: 0.1660 validation: 0.970000\n",
            "Iter-38900 loss: 0.1794 validation: 0.970000\n",
            "Iter-39000 loss: 0.1906 validation: 0.970000\n",
            "Iter-39100 loss: 0.2384 validation: 0.970000\n",
            "Iter-39200 loss: 0.2041 validation: 0.970000\n",
            "Iter-39300 loss: 0.2217 validation: 0.970000\n",
            "Iter-39400 loss: 0.1856 validation: 0.970000\n",
            "Iter-39500 loss: 0.1825 validation: 0.970000\n",
            "Iter-39600 loss: 0.2648 validation: 0.970000\n",
            "Iter-39700 loss: 0.2073 validation: 0.970000\n",
            "Iter-39800 loss: 0.2362 validation: 0.970000\n",
            "Iter-39900 loss: 0.1395 validation: 0.970000\n",
            "Iter-40000 loss: 0.2844 validation: 0.970000\n",
            "Iter-40100 loss: 0.2374 validation: 0.970000\n",
            "Iter-40200 loss: 0.2094 validation: 0.970000\n",
            "Iter-40300 loss: 0.1508 validation: 0.970000\n",
            "Iter-40400 loss: 0.2010 validation: 0.970000\n",
            "Iter-40500 loss: 0.1461 validation: 0.970000\n",
            "Iter-40600 loss: 0.1644 validation: 0.970000\n",
            "Iter-40700 loss: 0.1470 validation: 0.970000\n",
            "Iter-40800 loss: 0.1803 validation: 0.970000\n",
            "Iter-40900 loss: 0.2238 validation: 0.970000\n",
            "Iter-41000 loss: 0.1550 validation: 0.970000\n",
            "Iter-41100 loss: 0.1655 validation: 0.970000\n",
            "Iter-41200 loss: 0.2406 validation: 0.970000\n",
            "Iter-41300 loss: 0.1771 validation: 0.970000\n",
            "Iter-41400 loss: 0.1743 validation: 0.970000\n",
            "Iter-41500 loss: 0.2271 validation: 0.970000\n",
            "Iter-41600 loss: 0.1756 validation: 0.970000\n",
            "Iter-41700 loss: 0.1196 validation: 0.970000\n",
            "Iter-41800 loss: 0.2322 validation: 0.970000\n",
            "Iter-41900 loss: 0.1399 validation: 0.970000\n",
            "Iter-42000 loss: 0.1722 validation: 0.970000\n",
            "Iter-42100 loss: 0.2046 validation: 0.970000\n",
            "Iter-42200 loss: 0.1461 validation: 0.970000\n",
            "Iter-42300 loss: 0.2059 validation: 0.970000\n",
            "Iter-42400 loss: 0.1529 validation: 0.970000\n",
            "Iter-42500 loss: 0.2436 validation: 0.970000\n",
            "Iter-42600 loss: 0.1877 validation: 0.970000\n",
            "Iter-42700 loss: 0.1861 validation: 0.970000\n",
            "Iter-42800 loss: 0.1422 validation: 0.970000\n",
            "Iter-42900 loss: 0.2246 validation: 0.970000\n",
            "Iter-43000 loss: 0.1895 validation: 0.970000\n",
            "Iter-43100 loss: 0.1365 validation: 0.970000\n",
            "Iter-43200 loss: 0.1595 validation: 0.970000\n",
            "Iter-43300 loss: 0.2146 validation: 0.970000\n",
            "Iter-43400 loss: 0.2026 validation: 0.970000\n",
            "Iter-43500 loss: 0.1864 validation: 0.970000\n",
            "Iter-43600 loss: 0.2164 validation: 0.970000\n",
            "Iter-43700 loss: 0.2057 validation: 0.970000\n",
            "Iter-43800 loss: 0.1612 validation: 0.970000\n",
            "Iter-43900 loss: 0.2089 validation: 0.970000\n",
            "Iter-44000 loss: 0.2019 validation: 0.970000\n",
            "Iter-44100 loss: 0.1475 validation: 0.970000\n",
            "Iter-44200 loss: 0.1575 validation: 0.970000\n",
            "Iter-44300 loss: 0.1780 validation: 0.970000\n",
            "Iter-44400 loss: 0.1842 validation: 0.970000\n",
            "Iter-44500 loss: 0.1280 validation: 0.970000\n",
            "Iter-44600 loss: 0.1471 validation: 0.970000\n",
            "Iter-44700 loss: 0.1578 validation: 0.970000\n",
            "Iter-44800 loss: 0.1896 validation: 0.970000\n",
            "Iter-44900 loss: 0.1379 validation: 0.970000\n",
            "Iter-45000 loss: 0.1649 validation: 0.970000\n",
            "Iter-45100 loss: 0.1445 validation: 0.970000\n",
            "Iter-45200 loss: 0.1479 validation: 0.970000\n",
            "Iter-45300 loss: 0.1974 validation: 0.970000\n",
            "Iter-45400 loss: 0.1837 validation: 0.970000\n",
            "Iter-45500 loss: 0.1319 validation: 0.970000\n",
            "Iter-45600 loss: 0.1614 validation: 0.970000\n",
            "Iter-45700 loss: 0.1842 validation: 0.970000\n",
            "Iter-45800 loss: 0.1991 validation: 0.970000\n",
            "Iter-45900 loss: 0.1568 validation: 0.970000\n",
            "Iter-46000 loss: 0.1942 validation: 0.970000\n",
            "Iter-46100 loss: 0.2015 validation: 0.970000\n",
            "Iter-46200 loss: 0.1399 validation: 0.970000\n",
            "Iter-46300 loss: 0.1630 validation: 0.970000\n",
            "Iter-46400 loss: 0.1562 validation: 0.970000\n",
            "Iter-46500 loss: 0.1698 validation: 0.970000\n",
            "Iter-46600 loss: 0.1606 validation: 0.970000\n",
            "Iter-46700 loss: 0.1923 validation: 0.970000\n",
            "Iter-46800 loss: 0.1792 validation: 0.970000\n",
            "Iter-46900 loss: 0.1433 validation: 0.970000\n",
            "Iter-47000 loss: 0.2004 validation: 0.970000\n",
            "Iter-47100 loss: 0.1666 validation: 0.970000\n",
            "Iter-47200 loss: 0.1580 validation: 0.970000\n",
            "Iter-47300 loss: 0.1514 validation: 0.970000\n",
            "Iter-47400 loss: 0.1534 validation: 0.970000\n",
            "Iter-47500 loss: 0.1505 validation: 0.970000\n",
            "Iter-47600 loss: 0.1326 validation: 0.970000\n",
            "Iter-47700 loss: 0.2029 validation: 0.970000\n",
            "Iter-47800 loss: 0.1952 validation: 0.970000\n",
            "Iter-47900 loss: 0.1683 validation: 0.970000\n",
            "Iter-48000 loss: 0.1538 validation: 0.970000\n",
            "Iter-48100 loss: 0.1732 validation: 0.980000\n",
            "Iter-48200 loss: 0.1505 validation: 0.970000\n",
            "Iter-48300 loss: 0.1833 validation: 0.970000\n",
            "Iter-48400 loss: 0.1776 validation: 0.970000\n",
            "Iter-48500 loss: 0.1543 validation: 0.970000\n",
            "Iter-48600 loss: 0.1794 validation: 0.970000\n",
            "Iter-48700 loss: 0.1568 validation: 0.970000\n",
            "Iter-48800 loss: 0.1438 validation: 0.970000\n",
            "Iter-48900 loss: 0.1429 validation: 0.970000\n",
            "Iter-49000 loss: 0.1822 validation: 0.980000\n",
            "Iter-49100 loss: 0.1645 validation: 0.970000\n",
            "Iter-49200 loss: 0.1281 validation: 0.970000\n",
            "Iter-49300 loss: 0.1940 validation: 0.970000\n",
            "Iter-49400 loss: 0.1293 validation: 0.970000\n",
            "Iter-49500 loss: 0.1576 validation: 0.970000\n",
            "Iter-49600 loss: 0.1437 validation: 0.970000\n",
            "Iter-49700 loss: 0.1794 validation: 0.970000\n",
            "Iter-49800 loss: 0.1186 validation: 0.970000\n",
            "Iter-49900 loss: 0.1952 validation: 0.970000\n",
            "Iter-50000 loss: 0.1972 validation: 0.970000\n",
            "\n",
            "Best max_norm:  4.0 best dropout rate for input:  0.85 best dropout rate for hidden layers:  0.5 and result accuracy is:  0.9799\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NdQxPIZ6vC-W"
      },
      "source": [
        "Dropout NN + max-norm constraint + ReLU + 3 layers + 2048 hidden units (100,000 weight updates)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HqDUJHXmKDly",
        "outputId": "1acbd645-ef4e-4b59-9643-5da718bde7ef"
      },
      "source": [
        "model = 'ff'\n",
        "n_iter = 100000\n",
        "alpha = 1e-3\n",
        "mb_size = 64\n",
        "n_experiment = 1\n",
        "reg = 1e-5\n",
        "print_after = 100\n",
        "p_dropout = [0.8,0.5] #if nlayer=n, len(p_dropout) = n-1\n",
        "loss = 'cross_ent'\n",
        "nonlin = 'relu'\n",
        "solver = 'sgd'\n",
        "max_norm = 4\n",
        "nlayer = 3\n",
        "\n",
        "# Define default candidates\n",
        "candidates = np.array([[3, 3.5, 4],[0.795, 0.8, 0.85], [0.495, 0.5, 0.55]]) # first hyper-param as max-norm, second as drop-out rate for input layer, third as drop-out rate for hidden layers\n",
        "\n",
        "run_mnist(2048, model, n_iter, alpha, mb_size, n_experiment, reg, print_after, p_dropout, loss, nonlin, solver, max_norm, x_train, x_validation, x_test, y_train, y_validation, y_test, candidates, nlayer)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Experimenting on sgd\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  3.0 input drop out rate:  0.495 hidden layer drop out rate:  0.795\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.9158 validation: 0.790000\n",
            "Iter-200 loss: 1.7527 validation: 0.820000\n",
            "Iter-300 loss: 1.5620 validation: 0.850000\n",
            "Iter-400 loss: 1.5680 validation: 0.850000\n",
            "Iter-500 loss: 1.5221 validation: 0.870000\n",
            "\n",
            "Mean accuracy: 0.8515, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  3.0 input drop out rate:  0.495 hidden layer drop out rate:  0.8\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.9867 validation: 0.800000\n",
            "Iter-200 loss: 1.8086 validation: 0.830000\n",
            "Iter-300 loss: 1.6399 validation: 0.850000\n",
            "Iter-400 loss: 1.5575 validation: 0.860000\n",
            "Iter-500 loss: 1.4181 validation: 0.870000\n",
            "\n",
            "Mean accuracy: 0.8498, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  3.0 input drop out rate:  0.495 hidden layer drop out rate:  0.85\n",
            "Experiment-1\n",
            "Iter-100 loss: 2.0230 validation: 0.770000\n",
            "Iter-200 loss: 1.6704 validation: 0.830000\n",
            "Iter-300 loss: 1.6546 validation: 0.860000\n",
            "Iter-400 loss: 1.5841 validation: 0.860000\n",
            "Iter-500 loss: 1.4593 validation: 0.870000\n",
            "\n",
            "Mean accuracy: 0.8583, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  3.0 input drop out rate:  0.5 hidden layer drop out rate:  0.795\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.9748 validation: 0.810000\n",
            "Iter-200 loss: 1.7842 validation: 0.820000\n",
            "Iter-300 loss: 1.5669 validation: 0.830000\n",
            "Iter-400 loss: 1.4984 validation: 0.860000\n",
            "Iter-500 loss: 1.4789 validation: 0.870000\n",
            "\n",
            "Mean accuracy: 0.8546, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  3.0 input drop out rate:  0.5 hidden layer drop out rate:  0.8\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.9564 validation: 0.810000\n",
            "Iter-200 loss: 1.7944 validation: 0.850000\n",
            "Iter-300 loss: 1.6418 validation: 0.860000\n",
            "Iter-400 loss: 1.5770 validation: 0.870000\n",
            "Iter-500 loss: 1.5031 validation: 0.880000\n",
            "\n",
            "Mean accuracy: 0.8594, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  3.0 input drop out rate:  0.5 hidden layer drop out rate:  0.85\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.9638 validation: 0.800000\n",
            "Iter-200 loss: 1.7916 validation: 0.800000\n",
            "Iter-300 loss: 1.6038 validation: 0.850000\n",
            "Iter-400 loss: 1.5164 validation: 0.860000\n",
            "Iter-500 loss: 1.5455 validation: 0.860000\n",
            "\n",
            "Mean accuracy: 0.8528, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  3.0 input drop out rate:  0.55 hidden layer drop out rate:  0.795\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.9317 validation: 0.830000\n",
            "Iter-200 loss: 1.6977 validation: 0.830000\n",
            "Iter-300 loss: 1.5786 validation: 0.830000\n",
            "Iter-400 loss: 1.5427 validation: 0.870000\n",
            "Iter-500 loss: 1.4479 validation: 0.880000\n",
            "\n",
            "Mean accuracy: 0.8584, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  3.0 input drop out rate:  0.55 hidden layer drop out rate:  0.8\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.9218 validation: 0.810000\n",
            "Iter-200 loss: 1.7085 validation: 0.820000\n",
            "Iter-300 loss: 1.5526 validation: 0.840000\n",
            "Iter-400 loss: 1.5017 validation: 0.860000\n",
            "Iter-500 loss: 1.4919 validation: 0.890000\n",
            "\n",
            "Mean accuracy: 0.8584, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  3.0 input drop out rate:  0.55 hidden layer drop out rate:  0.85\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.8923 validation: 0.820000\n",
            "Iter-200 loss: 1.7219 validation: 0.840000\n",
            "Iter-300 loss: 1.6449 validation: 0.830000\n",
            "Iter-400 loss: 1.5358 validation: 0.860000\n",
            "Iter-500 loss: 1.4667 validation: 0.870000\n",
            "\n",
            "Mean accuracy: 0.8546, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  3.5 input drop out rate:  0.495 hidden layer drop out rate:  0.795\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.8722 validation: 0.820000\n",
            "Iter-200 loss: 1.5631 validation: 0.840000\n",
            "Iter-300 loss: 1.4722 validation: 0.870000\n",
            "Iter-400 loss: 1.3253 validation: 0.890000\n",
            "Iter-500 loss: 1.3895 validation: 0.890000\n",
            "\n",
            "Mean accuracy: 0.8634, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  3.5 input drop out rate:  0.495 hidden layer drop out rate:  0.8\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.8665 validation: 0.800000\n",
            "Iter-200 loss: 1.6186 validation: 0.830000\n",
            "Iter-300 loss: 1.4736 validation: 0.860000\n",
            "Iter-400 loss: 1.3510 validation: 0.870000\n",
            "Iter-500 loss: 1.2362 validation: 0.890000\n",
            "\n",
            "Mean accuracy: 0.8627, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  3.5 input drop out rate:  0.495 hidden layer drop out rate:  0.85\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.8463 validation: 0.820000\n",
            "Iter-200 loss: 1.5993 validation: 0.850000\n",
            "Iter-300 loss: 1.5167 validation: 0.860000\n",
            "Iter-400 loss: 1.3873 validation: 0.860000\n",
            "Iter-500 loss: 1.3180 validation: 0.890000\n",
            "\n",
            "Mean accuracy: 0.8648, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  3.5 input drop out rate:  0.5 hidden layer drop out rate:  0.795\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.8130 validation: 0.800000\n",
            "Iter-200 loss: 1.5058 validation: 0.840000\n",
            "Iter-300 loss: 1.4619 validation: 0.840000\n",
            "Iter-400 loss: 1.2689 validation: 0.850000\n",
            "Iter-500 loss: 1.2684 validation: 0.860000\n",
            "\n",
            "Mean accuracy: 0.8555, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  3.5 input drop out rate:  0.5 hidden layer drop out rate:  0.8\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.8690 validation: 0.790000\n",
            "Iter-200 loss: 1.5649 validation: 0.820000\n",
            "Iter-300 loss: 1.3916 validation: 0.830000\n",
            "Iter-400 loss: 1.4267 validation: 0.830000\n",
            "Iter-500 loss: 1.2929 validation: 0.870000\n",
            "\n",
            "Mean accuracy: 0.8562, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  3.5 input drop out rate:  0.5 hidden layer drop out rate:  0.85\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.8238 validation: 0.840000\n",
            "Iter-200 loss: 1.5033 validation: 0.830000\n",
            "Iter-300 loss: 1.4713 validation: 0.850000\n",
            "Iter-400 loss: 1.3646 validation: 0.890000\n",
            "Iter-500 loss: 1.2692 validation: 0.900000\n",
            "\n",
            "Mean accuracy: 0.8653, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  3.5 input drop out rate:  0.55 hidden layer drop out rate:  0.795\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.8246 validation: 0.770000\n",
            "Iter-200 loss: 1.5149 validation: 0.840000\n",
            "Iter-300 loss: 1.4250 validation: 0.850000\n",
            "Iter-400 loss: 1.3421 validation: 0.880000\n",
            "Iter-500 loss: 1.2374 validation: 0.890000\n",
            "\n",
            "Mean accuracy: 0.8628, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  3.5 input drop out rate:  0.55 hidden layer drop out rate:  0.8\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.8176 validation: 0.820000\n",
            "Iter-200 loss: 1.5600 validation: 0.870000\n",
            "Iter-300 loss: 1.3586 validation: 0.850000\n",
            "Iter-400 loss: 1.3551 validation: 0.850000\n",
            "Iter-500 loss: 1.3513 validation: 0.900000\n",
            "\n",
            "Mean accuracy: 0.8670, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  3.5 input drop out rate:  0.55 hidden layer drop out rate:  0.85\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.8333 validation: 0.810000\n",
            "Iter-200 loss: 1.5154 validation: 0.840000\n",
            "Iter-300 loss: 1.3939 validation: 0.850000\n",
            "Iter-400 loss: 1.3013 validation: 0.850000\n",
            "Iter-500 loss: 1.2142 validation: 0.890000\n",
            "\n",
            "Mean accuracy: 0.8656, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  4.0 input drop out rate:  0.495 hidden layer drop out rate:  0.795\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.7227 validation: 0.850000\n",
            "Iter-200 loss: 1.4311 validation: 0.850000\n",
            "Iter-300 loss: 1.2339 validation: 0.860000\n",
            "Iter-400 loss: 1.2211 validation: 0.860000\n",
            "Iter-500 loss: 1.0100 validation: 0.880000\n",
            "\n",
            "Mean accuracy: 0.8666, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  4.0 input drop out rate:  0.495 hidden layer drop out rate:  0.8\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.7094 validation: 0.770000\n",
            "Iter-200 loss: 1.4198 validation: 0.810000\n",
            "Iter-300 loss: 1.2613 validation: 0.850000\n",
            "Iter-400 loss: 1.2920 validation: 0.880000\n",
            "Iter-500 loss: 1.0436 validation: 0.900000\n",
            "\n",
            "Mean accuracy: 0.8711, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  4.0 input drop out rate:  0.495 hidden layer drop out rate:  0.85\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.7319 validation: 0.810000\n",
            "Iter-200 loss: 1.3896 validation: 0.840000\n",
            "Iter-300 loss: 1.3131 validation: 0.850000\n",
            "Iter-400 loss: 1.2213 validation: 0.880000\n",
            "Iter-500 loss: 1.0693 validation: 0.890000\n",
            "\n",
            "Mean accuracy: 0.8698, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  4.0 input drop out rate:  0.5 hidden layer drop out rate:  0.795\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.7127 validation: 0.840000\n",
            "Iter-200 loss: 1.4314 validation: 0.840000\n",
            "Iter-300 loss: 1.2527 validation: 0.870000\n",
            "Iter-400 loss: 1.1284 validation: 0.880000\n",
            "Iter-500 loss: 1.0736 validation: 0.890000\n",
            "\n",
            "Mean accuracy: 0.8704, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  4.0 input drop out rate:  0.5 hidden layer drop out rate:  0.8\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.7079 validation: 0.820000\n",
            "Iter-200 loss: 1.4348 validation: 0.850000\n",
            "Iter-300 loss: 1.3257 validation: 0.870000\n",
            "Iter-400 loss: 1.0549 validation: 0.890000\n",
            "Iter-500 loss: 1.0881 validation: 0.900000\n",
            "\n",
            "Mean accuracy: 0.8722, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  4.0 input drop out rate:  0.5 hidden layer drop out rate:  0.85\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.6887 validation: 0.790000\n",
            "Iter-200 loss: 1.5552 validation: 0.840000\n",
            "Iter-300 loss: 1.3717 validation: 0.840000\n",
            "Iter-400 loss: 1.1697 validation: 0.870000\n",
            "Iter-500 loss: 1.2051 validation: 0.900000\n",
            "\n",
            "Mean accuracy: 0.8673, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  4.0 input drop out rate:  0.55 hidden layer drop out rate:  0.795\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.6536 validation: 0.800000\n",
            "Iter-200 loss: 1.4079 validation: 0.860000\n",
            "Iter-300 loss: 1.3507 validation: 0.880000\n",
            "Iter-400 loss: 1.2348 validation: 0.880000\n",
            "Iter-500 loss: 1.0659 validation: 0.890000\n",
            "\n",
            "Mean accuracy: 0.8741, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  4.0 input drop out rate:  0.55 hidden layer drop out rate:  0.8\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.6623 validation: 0.830000\n",
            "Iter-200 loss: 1.3959 validation: 0.870000\n",
            "Iter-300 loss: 1.1998 validation: 0.870000\n",
            "Iter-400 loss: 1.2121 validation: 0.910000\n",
            "Iter-500 loss: 1.1321 validation: 0.910000\n",
            "\n",
            "Mean accuracy: 0.8756, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  4.0 input drop out rate:  0.55 hidden layer drop out rate:  0.85\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.6757 validation: 0.820000\n",
            "Iter-200 loss: 1.4041 validation: 0.830000\n",
            "Iter-300 loss: 1.2514 validation: 0.890000\n",
            "Iter-400 loss: 1.1050 validation: 0.890000\n",
            "Iter-500 loss: 1.0674 validation: 0.900000\n",
            "\n",
            "Mean accuracy: 0.8756, std: 0.0000\n",
            "\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.6894 validation: 0.820000\n",
            "Iter-200 loss: 1.3866 validation: 0.850000\n",
            "Iter-300 loss: 1.3895 validation: 0.860000\n",
            "Iter-400 loss: 1.1444 validation: 0.890000\n",
            "Iter-500 loss: 1.0972 validation: 0.890000\n",
            "Iter-600 loss: 1.0245 validation: 0.890000\n",
            "Iter-700 loss: 1.0123 validation: 0.910000\n",
            "Iter-800 loss: 0.9556 validation: 0.920000\n",
            "Iter-900 loss: 0.9350 validation: 0.930000\n",
            "Iter-1000 loss: 0.8729 validation: 0.950000\n",
            "Iter-1100 loss: 0.9136 validation: 0.950000\n",
            "Iter-1200 loss: 0.8147 validation: 0.950000\n",
            "Iter-1300 loss: 0.8009 validation: 0.950000\n",
            "Iter-1400 loss: 0.8393 validation: 0.950000\n",
            "Iter-1500 loss: 0.7404 validation: 0.950000\n",
            "Iter-1600 loss: 0.7450 validation: 0.950000\n",
            "Iter-1700 loss: 0.6840 validation: 0.950000\n",
            "Iter-1800 loss: 0.8287 validation: 0.950000\n",
            "Iter-1900 loss: 0.8512 validation: 0.950000\n",
            "Iter-2000 loss: 0.7396 validation: 0.950000\n",
            "Iter-2100 loss: 0.7966 validation: 0.950000\n",
            "Iter-2200 loss: 0.6645 validation: 0.950000\n",
            "Iter-2300 loss: 0.7600 validation: 0.950000\n",
            "Iter-2400 loss: 0.8119 validation: 0.950000\n",
            "Iter-2500 loss: 0.7781 validation: 0.950000\n",
            "Iter-2600 loss: 0.7807 validation: 0.950000\n",
            "Iter-2700 loss: 0.7641 validation: 0.950000\n",
            "Iter-2800 loss: 0.5524 validation: 0.950000\n",
            "Iter-2900 loss: 0.6189 validation: 0.950000\n",
            "Iter-3000 loss: 0.5903 validation: 0.950000\n",
            "Iter-3100 loss: 0.6238 validation: 0.950000\n",
            "Iter-3200 loss: 0.6584 validation: 0.950000\n",
            "Iter-3300 loss: 0.6284 validation: 0.950000\n",
            "Iter-3400 loss: 0.7499 validation: 0.950000\n",
            "Iter-3500 loss: 0.6104 validation: 0.950000\n",
            "Iter-3600 loss: 0.6311 validation: 0.950000\n",
            "Iter-3700 loss: 0.6526 validation: 0.950000\n",
            "Iter-3800 loss: 0.5681 validation: 0.950000\n",
            "Iter-3900 loss: 0.6255 validation: 0.950000\n",
            "Iter-4000 loss: 0.5920 validation: 0.950000\n",
            "Iter-4100 loss: 0.6922 validation: 0.950000\n",
            "Iter-4200 loss: 0.6013 validation: 0.950000\n",
            "Iter-4300 loss: 0.4820 validation: 0.950000\n",
            "Iter-4400 loss: 0.6569 validation: 0.950000\n",
            "Iter-4500 loss: 0.5712 validation: 0.950000\n",
            "Iter-4600 loss: 0.5219 validation: 0.950000\n",
            "Iter-4700 loss: 0.6109 validation: 0.950000\n",
            "Iter-4800 loss: 0.7076 validation: 0.950000\n",
            "Iter-4900 loss: 0.5310 validation: 0.950000\n",
            "Iter-5000 loss: 0.5283 validation: 0.950000\n",
            "Iter-5100 loss: 0.5605 validation: 0.950000\n",
            "Iter-5200 loss: 0.4947 validation: 0.950000\n",
            "Iter-5300 loss: 0.5083 validation: 0.950000\n",
            "Iter-5400 loss: 0.4726 validation: 0.950000\n",
            "Iter-5500 loss: 0.4831 validation: 0.950000\n",
            "Iter-5600 loss: 0.5307 validation: 0.950000\n",
            "Iter-5700 loss: 0.5602 validation: 0.950000\n",
            "Iter-5800 loss: 0.5358 validation: 0.950000\n",
            "Iter-5900 loss: 0.5504 validation: 0.950000\n",
            "Iter-6000 loss: 0.4611 validation: 0.950000\n",
            "Iter-6100 loss: 0.5154 validation: 0.960000\n",
            "Iter-6200 loss: 0.4887 validation: 0.950000\n",
            "Iter-6300 loss: 0.5714 validation: 0.950000\n",
            "Iter-6400 loss: 0.6370 validation: 0.950000\n",
            "Iter-6500 loss: 0.6548 validation: 0.950000\n",
            "Iter-6600 loss: 0.5801 validation: 0.950000\n",
            "Iter-6700 loss: 0.5170 validation: 0.950000\n",
            "Iter-6800 loss: 0.4744 validation: 0.950000\n",
            "Iter-6900 loss: 0.6111 validation: 0.950000\n",
            "Iter-7000 loss: 0.4472 validation: 0.950000\n",
            "Iter-7100 loss: 0.3561 validation: 0.960000\n",
            "Iter-7200 loss: 0.5529 validation: 0.960000\n",
            "Iter-7300 loss: 0.5208 validation: 0.970000\n",
            "Iter-7400 loss: 0.4796 validation: 0.950000\n",
            "Iter-7500 loss: 0.3963 validation: 0.960000\n",
            "Iter-7600 loss: 0.4252 validation: 0.970000\n",
            "Iter-7700 loss: 0.4226 validation: 0.960000\n",
            "Iter-7800 loss: 0.4815 validation: 0.960000\n",
            "Iter-7900 loss: 0.4857 validation: 0.960000\n",
            "Iter-8000 loss: 0.4953 validation: 0.970000\n",
            "Iter-8100 loss: 0.3554 validation: 0.970000\n",
            "Iter-8200 loss: 0.4011 validation: 0.970000\n",
            "Iter-8300 loss: 0.5201 validation: 0.970000\n",
            "Iter-8400 loss: 0.4101 validation: 0.970000\n",
            "Iter-8500 loss: 0.4487 validation: 0.970000\n",
            "Iter-8600 loss: 0.4425 validation: 0.970000\n",
            "Iter-8700 loss: 0.5125 validation: 0.970000\n",
            "Iter-8800 loss: 0.3706 validation: 0.970000\n",
            "Iter-8900 loss: 0.4241 validation: 0.970000\n",
            "Iter-9000 loss: 0.4387 validation: 0.970000\n",
            "Iter-9100 loss: 0.3585 validation: 0.970000\n",
            "Iter-9200 loss: 0.4972 validation: 0.970000\n",
            "Iter-9300 loss: 0.4794 validation: 0.970000\n",
            "Iter-9400 loss: 0.3680 validation: 0.970000\n",
            "Iter-9500 loss: 0.4115 validation: 0.970000\n",
            "Iter-9600 loss: 0.3544 validation: 0.970000\n",
            "Iter-9700 loss: 0.3710 validation: 0.970000\n",
            "Iter-9800 loss: 0.4709 validation: 0.970000\n",
            "Iter-9900 loss: 0.3270 validation: 0.970000\n",
            "Iter-10000 loss: 0.3593 validation: 0.970000\n",
            "Iter-10100 loss: 0.4653 validation: 0.970000\n",
            "Iter-10200 loss: 0.3936 validation: 0.970000\n",
            "Iter-10300 loss: 0.3584 validation: 0.970000\n",
            "Iter-10400 loss: 0.4055 validation: 0.970000\n",
            "Iter-10500 loss: 0.6538 validation: 0.970000\n",
            "Iter-10600 loss: 0.4440 validation: 0.970000\n",
            "Iter-10700 loss: 0.4119 validation: 0.970000\n",
            "Iter-10800 loss: 0.3076 validation: 0.970000\n",
            "Iter-10900 loss: 0.3531 validation: 0.970000\n",
            "Iter-11000 loss: 0.5162 validation: 0.970000\n",
            "Iter-11100 loss: 0.3712 validation: 0.970000\n",
            "Iter-11200 loss: 0.4149 validation: 0.970000\n",
            "Iter-11300 loss: 0.4468 validation: 0.970000\n",
            "Iter-11400 loss: 0.4343 validation: 0.970000\n",
            "Iter-11500 loss: 0.3716 validation: 0.970000\n",
            "Iter-11600 loss: 0.5042 validation: 0.970000\n",
            "Iter-11700 loss: 0.4459 validation: 0.970000\n",
            "Iter-11800 loss: 0.3664 validation: 0.970000\n",
            "Iter-11900 loss: 0.3600 validation: 0.970000\n",
            "Iter-12000 loss: 0.3931 validation: 0.970000\n",
            "Iter-12100 loss: 0.3775 validation: 0.970000\n",
            "Iter-12200 loss: 0.3485 validation: 0.970000\n",
            "Iter-12300 loss: 0.4393 validation: 0.970000\n",
            "Iter-12400 loss: 0.3556 validation: 0.970000\n",
            "Iter-12500 loss: 0.3266 validation: 0.970000\n",
            "Iter-12600 loss: 0.3164 validation: 0.970000\n",
            "Iter-12700 loss: 0.3689 validation: 0.970000\n",
            "Iter-12800 loss: 0.3158 validation: 0.970000\n",
            "Iter-12900 loss: 0.3311 validation: 0.970000\n",
            "Iter-13000 loss: 0.4017 validation: 0.970000\n",
            "Iter-13100 loss: 0.3955 validation: 0.970000\n",
            "Iter-13200 loss: 0.2662 validation: 0.970000\n",
            "Iter-13300 loss: 0.3309 validation: 0.970000\n",
            "Iter-13400 loss: 0.3343 validation: 0.970000\n",
            "Iter-13500 loss: 0.3144 validation: 0.970000\n",
            "Iter-13600 loss: 0.4231 validation: 0.970000\n",
            "Iter-13700 loss: 0.3884 validation: 0.970000\n",
            "Iter-13800 loss: 0.3611 validation: 0.970000\n",
            "Iter-13900 loss: 0.3733 validation: 0.970000\n",
            "Iter-14000 loss: 0.3695 validation: 0.970000\n",
            "Iter-14100 loss: 0.3104 validation: 0.970000\n",
            "Iter-14200 loss: 0.3541 validation: 0.970000\n",
            "Iter-14300 loss: 0.3440 validation: 0.970000\n",
            "Iter-14400 loss: 0.3249 validation: 0.970000\n",
            "Iter-14500 loss: 0.3126 validation: 0.970000\n",
            "Iter-14600 loss: 0.3372 validation: 0.970000\n",
            "Iter-14700 loss: 0.2928 validation: 0.970000\n",
            "Iter-14800 loss: 0.3034 validation: 0.970000\n",
            "Iter-14900 loss: 0.4383 validation: 0.970000\n",
            "Iter-15000 loss: 0.3568 validation: 0.970000\n",
            "Iter-15100 loss: 0.3200 validation: 0.970000\n",
            "Iter-15200 loss: 0.2821 validation: 0.970000\n",
            "Iter-15300 loss: 0.3577 validation: 0.970000\n",
            "Iter-15400 loss: 0.2949 validation: 0.970000\n",
            "Iter-15500 loss: 0.2577 validation: 0.970000\n",
            "Iter-15600 loss: 0.3121 validation: 0.970000\n",
            "Iter-15700 loss: 0.3138 validation: 0.970000\n",
            "Iter-15800 loss: 0.3446 validation: 0.970000\n",
            "Iter-15900 loss: 0.2968 validation: 0.970000\n",
            "Iter-16000 loss: 0.2908 validation: 0.970000\n",
            "Iter-16100 loss: 0.2757 validation: 0.970000\n",
            "Iter-16200 loss: 0.3479 validation: 0.970000\n",
            "Iter-16300 loss: 0.3307 validation: 0.970000\n",
            "Iter-16400 loss: 0.3075 validation: 0.970000\n",
            "Iter-16500 loss: 0.3209 validation: 0.970000\n",
            "Iter-16600 loss: 0.3505 validation: 0.970000\n",
            "Iter-16700 loss: 0.3954 validation: 0.970000\n",
            "Iter-16800 loss: 0.3354 validation: 0.970000\n",
            "Iter-16900 loss: 0.3632 validation: 0.970000\n",
            "Iter-17000 loss: 0.2534 validation: 0.970000\n",
            "Iter-17100 loss: 0.3564 validation: 0.970000\n",
            "Iter-17200 loss: 0.2920 validation: 0.970000\n",
            "Iter-17300 loss: 0.3491 validation: 0.970000\n",
            "Iter-17400 loss: 0.3055 validation: 0.970000\n",
            "Iter-17500 loss: 0.3657 validation: 0.970000\n",
            "Iter-17600 loss: 0.3162 validation: 0.970000\n",
            "Iter-17700 loss: 0.2493 validation: 0.970000\n",
            "Iter-17800 loss: 0.2944 validation: 0.970000\n",
            "Iter-17900 loss: 0.3573 validation: 0.970000\n",
            "Iter-18000 loss: 0.2812 validation: 0.970000\n",
            "Iter-18100 loss: 0.2831 validation: 0.970000\n",
            "Iter-18200 loss: 0.3238 validation: 0.970000\n",
            "Iter-18300 loss: 0.2558 validation: 0.970000\n",
            "Iter-18400 loss: 0.2736 validation: 0.970000\n",
            "Iter-18500 loss: 0.3458 validation: 0.970000\n",
            "Iter-18600 loss: 0.3251 validation: 0.970000\n",
            "Iter-18700 loss: 0.2872 validation: 0.970000\n",
            "Iter-18800 loss: 0.3177 validation: 0.970000\n",
            "Iter-18900 loss: 0.2751 validation: 0.970000\n",
            "Iter-19000 loss: 0.3101 validation: 0.970000\n",
            "Iter-19100 loss: 0.2864 validation: 0.970000\n",
            "Iter-19200 loss: 0.2452 validation: 0.970000\n",
            "Iter-19300 loss: 0.2746 validation: 0.970000\n",
            "Iter-19400 loss: 0.2367 validation: 0.970000\n",
            "Iter-19500 loss: 0.2838 validation: 0.970000\n",
            "Iter-19600 loss: 0.2778 validation: 0.970000\n",
            "Iter-19700 loss: 0.2801 validation: 0.970000\n",
            "Iter-19800 loss: 0.2768 validation: 0.970000\n",
            "Iter-19900 loss: 0.2791 validation: 0.970000\n",
            "Iter-20000 loss: 0.2724 validation: 0.970000\n",
            "Iter-20100 loss: 0.2973 validation: 0.970000\n",
            "Iter-20200 loss: 0.2739 validation: 0.970000\n",
            "Iter-20300 loss: 0.3671 validation: 0.970000\n",
            "Iter-20400 loss: 0.3023 validation: 0.970000\n",
            "Iter-20500 loss: 0.2275 validation: 0.970000\n",
            "Iter-20600 loss: 0.2310 validation: 0.970000\n",
            "Iter-20700 loss: 0.3690 validation: 0.970000\n",
            "Iter-20800 loss: 0.3236 validation: 0.970000\n",
            "Iter-20900 loss: 0.3073 validation: 0.970000\n",
            "Iter-21000 loss: 0.2477 validation: 0.970000\n",
            "Iter-21100 loss: 0.2373 validation: 0.970000\n",
            "Iter-21200 loss: 0.2143 validation: 0.970000\n",
            "Iter-21300 loss: 0.3322 validation: 0.970000\n",
            "Iter-21400 loss: 0.2633 validation: 0.970000\n",
            "Iter-21500 loss: 0.2948 validation: 0.970000\n",
            "Iter-21600 loss: 0.2474 validation: 0.970000\n",
            "Iter-21700 loss: 0.3383 validation: 0.970000\n",
            "Iter-21800 loss: 0.1877 validation: 0.970000\n",
            "Iter-21900 loss: 0.2458 validation: 0.970000\n",
            "Iter-22000 loss: 0.2905 validation: 0.970000\n",
            "Iter-22100 loss: 0.3289 validation: 0.970000\n",
            "Iter-22200 loss: 0.2538 validation: 0.970000\n",
            "Iter-22300 loss: 0.2607 validation: 0.970000\n",
            "Iter-22400 loss: 0.2787 validation: 0.970000\n",
            "Iter-22500 loss: 0.3330 validation: 0.970000\n",
            "Iter-22600 loss: 0.2836 validation: 0.970000\n",
            "Iter-22700 loss: 0.2909 validation: 0.970000\n",
            "Iter-22800 loss: 0.2278 validation: 0.970000\n",
            "Iter-22900 loss: 0.3502 validation: 0.970000\n",
            "Iter-23000 loss: 0.2208 validation: 0.970000\n",
            "Iter-23100 loss: 0.2454 validation: 0.970000\n",
            "Iter-23200 loss: 0.2563 validation: 0.970000\n",
            "Iter-23300 loss: 0.2247 validation: 0.970000\n",
            "Iter-23400 loss: 0.2596 validation: 0.970000\n",
            "Iter-23500 loss: 0.2318 validation: 0.970000\n",
            "Iter-23600 loss: 0.2325 validation: 0.970000\n",
            "Iter-23700 loss: 0.2666 validation: 0.970000\n",
            "Iter-23800 loss: 0.2666 validation: 0.970000\n",
            "Iter-23900 loss: 0.2447 validation: 0.970000\n",
            "Iter-24000 loss: 0.2052 validation: 0.970000\n",
            "Iter-24100 loss: 0.2652 validation: 0.970000\n",
            "Iter-24200 loss: 0.2365 validation: 0.970000\n",
            "Iter-24300 loss: 0.2474 validation: 0.970000\n",
            "Iter-24400 loss: 0.2758 validation: 0.970000\n",
            "Iter-24500 loss: 0.2099 validation: 0.970000\n",
            "Iter-24600 loss: 0.2373 validation: 0.970000\n",
            "Iter-24700 loss: 0.2007 validation: 0.970000\n",
            "Iter-24800 loss: 0.2461 validation: 0.970000\n",
            "Iter-24900 loss: 0.2441 validation: 0.970000\n",
            "Iter-25000 loss: 0.2257 validation: 0.970000\n",
            "Iter-25100 loss: 0.2798 validation: 0.970000\n",
            "Iter-25200 loss: 0.2969 validation: 0.970000\n",
            "Iter-25300 loss: 0.2983 validation: 0.970000\n",
            "Iter-25400 loss: 0.2600 validation: 0.970000\n",
            "Iter-25500 loss: 0.2220 validation: 0.970000\n",
            "Iter-25600 loss: 0.2527 validation: 0.970000\n",
            "Iter-25700 loss: 0.2413 validation: 0.970000\n",
            "Iter-25800 loss: 0.2995 validation: 0.970000\n",
            "Iter-25900 loss: 0.2873 validation: 0.970000\n",
            "Iter-26000 loss: 0.3942 validation: 0.970000\n",
            "Iter-26100 loss: 0.2801 validation: 0.970000\n",
            "Iter-26200 loss: 0.2581 validation: 0.970000\n",
            "Iter-26300 loss: 0.2369 validation: 0.970000\n",
            "Iter-26400 loss: 0.2438 validation: 0.970000\n",
            "Iter-26500 loss: 0.2827 validation: 0.970000\n",
            "Iter-26600 loss: 0.2421 validation: 0.970000\n",
            "Iter-26700 loss: 0.3271 validation: 0.970000\n",
            "Iter-26800 loss: 0.2854 validation: 0.970000\n",
            "Iter-26900 loss: 0.2036 validation: 0.970000\n",
            "Iter-27000 loss: 0.3061 validation: 0.970000\n",
            "Iter-27100 loss: 0.2371 validation: 0.970000\n",
            "Iter-27200 loss: 0.2179 validation: 0.970000\n",
            "Iter-27300 loss: 0.2395 validation: 0.970000\n",
            "Iter-27400 loss: 0.2233 validation: 0.970000\n",
            "Iter-27500 loss: 0.2149 validation: 0.970000\n",
            "Iter-27600 loss: 0.2208 validation: 0.970000\n",
            "Iter-27700 loss: 0.1875 validation: 0.970000\n",
            "Iter-27800 loss: 0.2405 validation: 0.970000\n",
            "Iter-27900 loss: 0.2198 validation: 0.970000\n",
            "Iter-28000 loss: 0.2213 validation: 0.970000\n",
            "Iter-28100 loss: 0.2537 validation: 0.970000\n",
            "Iter-28200 loss: 0.2263 validation: 0.970000\n",
            "Iter-28300 loss: 0.3016 validation: 0.970000\n",
            "Iter-28400 loss: 0.2325 validation: 0.970000\n",
            "Iter-28500 loss: 0.2164 validation: 0.970000\n",
            "Iter-28600 loss: 0.2192 validation: 0.970000\n",
            "Iter-28700 loss: 0.2414 validation: 0.970000\n",
            "Iter-28800 loss: 0.2160 validation: 0.970000\n",
            "Iter-28900 loss: 0.3089 validation: 0.970000\n",
            "Iter-29000 loss: 0.2859 validation: 0.970000\n",
            "Iter-29100 loss: 0.2446 validation: 0.970000\n",
            "Iter-29200 loss: 0.2116 validation: 0.970000\n",
            "Iter-29300 loss: 0.2652 validation: 0.970000\n",
            "Iter-29400 loss: 0.2609 validation: 0.970000\n",
            "Iter-29500 loss: 0.2963 validation: 0.970000\n",
            "Iter-29600 loss: 0.2304 validation: 0.970000\n",
            "Iter-29700 loss: 0.2194 validation: 0.970000\n",
            "Iter-29800 loss: 0.2222 validation: 0.970000\n",
            "Iter-29900 loss: 0.2237 validation: 0.970000\n",
            "Iter-30000 loss: 0.2790 validation: 0.970000\n",
            "Iter-30100 loss: 0.2039 validation: 0.970000\n",
            "Iter-30200 loss: 0.1982 validation: 0.970000\n",
            "Iter-30300 loss: 0.2945 validation: 0.970000\n",
            "Iter-30400 loss: 0.2205 validation: 0.970000\n",
            "Iter-30500 loss: 0.2491 validation: 0.970000\n",
            "Iter-30600 loss: 0.2424 validation: 0.970000\n",
            "Iter-30700 loss: 0.2156 validation: 0.970000\n",
            "Iter-30800 loss: 0.1915 validation: 0.970000\n",
            "Iter-30900 loss: 0.2513 validation: 0.970000\n",
            "Iter-31000 loss: 0.2456 validation: 0.970000\n",
            "Iter-31100 loss: 0.2204 validation: 0.970000\n",
            "Iter-31200 loss: 0.1855 validation: 0.970000\n",
            "Iter-31300 loss: 0.2297 validation: 0.970000\n",
            "Iter-31400 loss: 0.1805 validation: 0.970000\n",
            "Iter-31500 loss: 0.2540 validation: 0.970000\n",
            "Iter-31600 loss: 0.2199 validation: 0.970000\n",
            "Iter-31700 loss: 0.1631 validation: 0.970000\n",
            "Iter-31800 loss: 0.1830 validation: 0.970000\n",
            "Iter-31900 loss: 0.2808 validation: 0.970000\n",
            "Iter-32000 loss: 0.2370 validation: 0.970000\n",
            "Iter-32100 loss: 0.1884 validation: 0.970000\n",
            "Iter-32200 loss: 0.2462 validation: 0.970000\n",
            "Iter-32300 loss: 0.2052 validation: 0.970000\n",
            "Iter-32400 loss: 0.2559 validation: 0.970000\n",
            "Iter-32500 loss: 0.2598 validation: 0.970000\n",
            "Iter-32600 loss: 0.2883 validation: 0.970000\n",
            "Iter-32700 loss: 0.2447 validation: 0.970000\n",
            "Iter-32800 loss: 0.1890 validation: 0.970000\n",
            "Iter-32900 loss: 0.1850 validation: 0.970000\n",
            "Iter-33000 loss: 0.1711 validation: 0.970000\n",
            "Iter-33100 loss: 0.1657 validation: 0.970000\n",
            "Iter-33200 loss: 0.2213 validation: 0.970000\n",
            "Iter-33300 loss: 0.1589 validation: 0.970000\n",
            "Iter-33400 loss: 0.2216 validation: 0.970000\n",
            "Iter-33500 loss: 0.1989 validation: 0.970000\n",
            "Iter-33600 loss: 0.2131 validation: 0.970000\n",
            "Iter-33700 loss: 0.2496 validation: 0.970000\n",
            "Iter-33800 loss: 0.3345 validation: 0.970000\n",
            "Iter-33900 loss: 0.1997 validation: 0.970000\n",
            "Iter-34000 loss: 0.2011 validation: 0.970000\n",
            "Iter-34100 loss: 0.2114 validation: 0.970000\n",
            "Iter-34200 loss: 0.2372 validation: 0.970000\n",
            "Iter-34300 loss: 0.1876 validation: 0.970000\n",
            "Iter-34400 loss: 0.2292 validation: 0.970000\n",
            "Iter-34500 loss: 0.2444 validation: 0.970000\n",
            "Iter-34600 loss: 0.1840 validation: 0.970000\n",
            "Iter-34700 loss: 0.2237 validation: 0.970000\n",
            "Iter-34800 loss: 0.2194 validation: 0.970000\n",
            "Iter-34900 loss: 0.2031 validation: 0.970000\n",
            "Iter-35000 loss: 0.2152 validation: 0.970000\n",
            "Iter-35100 loss: 0.2485 validation: 0.970000\n",
            "Iter-35200 loss: 0.3034 validation: 0.970000\n",
            "Iter-35300 loss: 0.2735 validation: 0.970000\n",
            "Iter-35400 loss: 0.2002 validation: 0.970000\n",
            "Iter-35500 loss: 0.1702 validation: 0.970000\n",
            "Iter-35600 loss: 0.1912 validation: 0.970000\n",
            "Iter-35700 loss: 0.1819 validation: 0.970000\n",
            "Iter-35800 loss: 0.2475 validation: 0.970000\n",
            "Iter-35900 loss: 0.1676 validation: 0.970000\n",
            "Iter-36000 loss: 0.1990 validation: 0.970000\n",
            "Iter-36100 loss: 0.2486 validation: 0.970000\n",
            "Iter-36200 loss: 0.2150 validation: 0.970000\n",
            "Iter-36300 loss: 0.1824 validation: 0.970000\n",
            "Iter-36400 loss: 0.2508 validation: 0.970000\n",
            "Iter-36500 loss: 0.2022 validation: 0.970000\n",
            "Iter-36600 loss: 0.2438 validation: 0.970000\n",
            "Iter-36700 loss: 0.2524 validation: 0.970000\n",
            "Iter-36800 loss: 0.2423 validation: 0.970000\n",
            "Iter-36900 loss: 0.1959 validation: 0.970000\n",
            "Iter-37000 loss: 0.1985 validation: 0.970000\n",
            "Iter-37100 loss: 0.2349 validation: 0.970000\n",
            "Iter-37200 loss: 0.2195 validation: 0.970000\n",
            "Iter-37300 loss: 0.3189 validation: 0.970000\n",
            "Iter-37400 loss: 0.2233 validation: 0.970000\n",
            "Iter-37500 loss: 0.1913 validation: 0.970000\n",
            "Iter-37600 loss: 0.1860 validation: 0.970000\n",
            "Iter-37700 loss: 0.2743 validation: 0.970000\n",
            "Iter-37800 loss: 0.2203 validation: 0.970000\n",
            "Iter-37900 loss: 0.1609 validation: 0.970000\n",
            "Iter-38000 loss: 0.2228 validation: 0.970000\n",
            "Iter-38100 loss: 0.1696 validation: 0.970000\n",
            "Iter-38200 loss: 0.2409 validation: 0.970000\n",
            "Iter-38300 loss: 0.1911 validation: 0.970000\n",
            "Iter-38400 loss: 0.1754 validation: 0.970000\n",
            "Iter-38500 loss: 0.1881 validation: 0.970000\n",
            "Iter-38600 loss: 0.2203 validation: 0.970000\n",
            "Iter-38700 loss: 0.1881 validation: 0.970000\n",
            "Iter-38800 loss: 0.2173 validation: 0.970000\n",
            "Iter-38900 loss: 0.1617 validation: 0.970000\n",
            "Iter-39000 loss: 0.2271 validation: 0.970000\n",
            "Iter-39100 loss: 0.2283 validation: 0.980000\n",
            "Iter-39200 loss: 0.1942 validation: 0.970000\n",
            "Iter-39300 loss: 0.2320 validation: 0.970000\n",
            "Iter-39400 loss: 0.2375 validation: 0.970000\n",
            "Iter-39500 loss: 0.2363 validation: 0.970000\n",
            "Iter-39600 loss: 0.1799 validation: 0.970000\n",
            "Iter-39700 loss: 0.2316 validation: 0.970000\n",
            "Iter-39800 loss: 0.1689 validation: 0.970000\n",
            "Iter-39900 loss: 0.1695 validation: 0.970000\n",
            "Iter-40000 loss: 0.1780 validation: 0.970000\n",
            "Iter-40100 loss: 0.2002 validation: 0.970000\n",
            "Iter-40200 loss: 0.2373 validation: 0.970000\n",
            "Iter-40300 loss: 0.2035 validation: 0.970000\n",
            "Iter-40400 loss: 0.1948 validation: 0.970000\n",
            "Iter-40500 loss: 0.2362 validation: 0.970000\n",
            "Iter-40600 loss: 0.2038 validation: 0.970000\n",
            "Iter-40700 loss: 0.1825 validation: 0.970000\n",
            "Iter-40800 loss: 0.2175 validation: 0.970000\n",
            "Iter-40900 loss: 0.1693 validation: 0.970000\n",
            "Iter-41000 loss: 0.1569 validation: 0.970000\n",
            "Iter-41100 loss: 0.1734 validation: 0.980000\n",
            "Iter-41200 loss: 0.2489 validation: 0.970000\n",
            "Iter-41300 loss: 0.1735 validation: 0.970000\n",
            "Iter-41400 loss: 0.1766 validation: 0.970000\n",
            "Iter-41500 loss: 0.2082 validation: 0.970000\n",
            "Iter-41600 loss: 0.2168 validation: 0.970000\n",
            "Iter-41700 loss: 0.1497 validation: 0.970000\n",
            "Iter-41800 loss: 0.1847 validation: 0.970000\n",
            "Iter-41900 loss: 0.1612 validation: 0.970000\n",
            "Iter-42000 loss: 0.2013 validation: 0.970000\n",
            "Iter-42100 loss: 0.1813 validation: 0.970000\n",
            "Iter-42200 loss: 0.1818 validation: 0.970000\n",
            "Iter-42300 loss: 0.2597 validation: 0.970000\n",
            "Iter-42400 loss: 0.1712 validation: 0.970000\n",
            "Iter-42500 loss: 0.1992 validation: 0.970000\n",
            "Iter-42600 loss: 0.1790 validation: 0.970000\n",
            "Iter-42700 loss: 0.1857 validation: 0.970000\n",
            "Iter-42800 loss: 0.1865 validation: 0.970000\n",
            "Iter-42900 loss: 0.1716 validation: 0.970000\n",
            "Iter-43000 loss: 0.1643 validation: 0.970000\n",
            "Iter-43100 loss: 0.2083 validation: 0.970000\n",
            "Iter-43200 loss: 0.2127 validation: 0.970000\n",
            "Iter-43300 loss: 0.2192 validation: 0.970000\n",
            "Iter-43400 loss: 0.1393 validation: 0.970000\n",
            "Iter-43500 loss: 0.2509 validation: 0.970000\n",
            "Iter-43600 loss: 0.2014 validation: 0.970000\n",
            "Iter-43700 loss: 0.1615 validation: 0.970000\n",
            "Iter-43800 loss: 0.2080 validation: 0.970000\n",
            "Iter-43900 loss: 0.2117 validation: 0.970000\n",
            "Iter-44000 loss: 0.2030 validation: 0.970000\n",
            "Iter-44100 loss: 0.2143 validation: 0.970000\n",
            "Iter-44200 loss: 0.2022 validation: 0.970000\n",
            "Iter-44300 loss: 0.2003 validation: 0.970000\n",
            "Iter-44400 loss: 0.2071 validation: 0.970000\n",
            "Iter-44500 loss: 0.1737 validation: 0.970000\n",
            "Iter-44600 loss: 0.1990 validation: 0.970000\n",
            "Iter-44700 loss: 0.2130 validation: 0.970000\n",
            "Iter-44800 loss: 0.1592 validation: 0.970000\n",
            "Iter-44900 loss: 0.1490 validation: 0.970000\n",
            "Iter-45000 loss: 0.2416 validation: 0.970000\n",
            "Iter-45100 loss: 0.2015 validation: 0.970000\n",
            "Iter-45200 loss: 0.2020 validation: 0.970000\n",
            "Iter-45300 loss: 0.1649 validation: 0.970000\n",
            "Iter-45400 loss: 0.1941 validation: 0.970000\n",
            "Iter-45500 loss: 0.1988 validation: 0.970000\n",
            "Iter-45600 loss: 0.2222 validation: 0.970000\n",
            "Iter-45700 loss: 0.1868 validation: 0.970000\n",
            "Iter-45800 loss: 0.2010 validation: 0.970000\n",
            "Iter-45900 loss: 0.1827 validation: 0.970000\n",
            "Iter-46000 loss: 0.2147 validation: 0.970000\n",
            "Iter-46100 loss: 0.1491 validation: 0.970000\n",
            "Iter-46200 loss: 0.2260 validation: 0.970000\n",
            "Iter-46300 loss: 0.1491 validation: 0.970000\n",
            "Iter-46400 loss: 0.1860 validation: 0.970000\n",
            "Iter-46500 loss: 0.1575 validation: 0.970000\n",
            "Iter-46600 loss: 0.1706 validation: 0.970000\n",
            "Iter-46700 loss: 0.1696 validation: 0.970000\n",
            "Iter-46800 loss: 0.1690 validation: 0.970000\n",
            "Iter-46900 loss: 0.1703 validation: 0.970000\n",
            "Iter-47000 loss: 0.2203 validation: 0.970000\n",
            "Iter-47100 loss: 0.2019 validation: 0.970000\n",
            "Iter-47200 loss: 0.1715 validation: 0.970000\n",
            "Iter-47300 loss: 0.1433 validation: 0.980000\n",
            "Iter-47400 loss: 0.1411 validation: 0.970000\n",
            "Iter-47500 loss: 0.1669 validation: 0.970000\n",
            "Iter-47600 loss: 0.1711 validation: 0.970000\n",
            "Iter-47700 loss: 0.1530 validation: 0.970000\n",
            "Iter-47800 loss: 0.1829 validation: 0.970000\n",
            "Iter-47900 loss: 0.1439 validation: 0.970000\n",
            "Iter-48000 loss: 0.1768 validation: 0.970000\n",
            "Iter-48100 loss: 0.2135 validation: 0.970000\n",
            "Iter-48200 loss: 0.2157 validation: 0.970000\n",
            "Iter-48300 loss: 0.1962 validation: 0.970000\n",
            "Iter-48400 loss: 0.2162 validation: 0.970000\n",
            "Iter-48500 loss: 0.2301 validation: 0.970000\n",
            "Iter-48600 loss: 0.1871 validation: 0.970000\n",
            "Iter-48700 loss: 0.1727 validation: 0.970000\n",
            "Iter-48800 loss: 0.1440 validation: 0.980000\n",
            "Iter-48900 loss: 0.1830 validation: 0.970000\n",
            "Iter-49000 loss: 0.1724 validation: 0.980000\n",
            "Iter-49100 loss: 0.1812 validation: 0.970000\n",
            "Iter-49200 loss: 0.1652 validation: 0.980000\n",
            "Iter-49300 loss: 0.2176 validation: 0.970000\n",
            "Iter-49400 loss: 0.2015 validation: 0.970000\n",
            "Iter-49500 loss: 0.1585 validation: 0.970000\n",
            "Iter-49600 loss: 0.2442 validation: 0.970000\n",
            "Iter-49700 loss: 0.1659 validation: 0.970000\n",
            "Iter-49800 loss: 0.1660 validation: 0.970000\n",
            "Iter-49900 loss: 0.1738 validation: 0.970000\n",
            "Iter-50000 loss: 0.2140 validation: 0.970000\n",
            "Iter-50100 loss: 0.1510 validation: 0.970000\n",
            "Iter-50200 loss: 0.2393 validation: 0.970000\n",
            "Iter-50300 loss: 0.1762 validation: 0.970000\n",
            "Iter-50400 loss: 0.1455 validation: 0.970000\n",
            "Iter-50500 loss: 0.1642 validation: 0.970000\n",
            "Iter-50600 loss: 0.1437 validation: 0.970000\n",
            "Iter-50700 loss: 0.1844 validation: 0.970000\n",
            "Iter-50800 loss: 0.1584 validation: 0.970000\n",
            "Iter-50900 loss: 0.1415 validation: 0.970000\n",
            "Iter-51000 loss: 0.2190 validation: 0.970000\n",
            "Iter-51100 loss: 0.1642 validation: 0.980000\n",
            "Iter-51200 loss: 0.1563 validation: 0.970000\n",
            "Iter-51300 loss: 0.2034 validation: 0.970000\n",
            "Iter-51400 loss: 0.1966 validation: 0.980000\n",
            "Iter-51500 loss: 0.1287 validation: 0.970000\n",
            "Iter-51600 loss: 0.1576 validation: 0.970000\n",
            "Iter-51700 loss: 0.1522 validation: 0.970000\n",
            "Iter-51800 loss: 0.1810 validation: 0.970000\n",
            "Iter-51900 loss: 0.2471 validation: 0.970000\n",
            "Iter-52000 loss: 0.1636 validation: 0.970000\n",
            "Iter-52100 loss: 0.1649 validation: 0.970000\n",
            "Iter-52200 loss: 0.2035 validation: 0.970000\n",
            "Iter-52300 loss: 0.1617 validation: 0.970000\n",
            "Iter-52400 loss: 0.1758 validation: 0.970000\n",
            "Iter-52500 loss: 0.1630 validation: 0.970000\n",
            "Iter-52600 loss: 0.1628 validation: 0.970000\n",
            "Iter-52700 loss: 0.1536 validation: 0.970000\n",
            "Iter-52800 loss: 0.1785 validation: 0.970000\n",
            "Iter-52900 loss: 0.1879 validation: 0.970000\n",
            "Iter-53000 loss: 0.1588 validation: 0.970000\n",
            "Iter-53100 loss: 0.1401 validation: 0.970000\n",
            "Iter-53200 loss: 0.1883 validation: 0.970000\n",
            "Iter-53300 loss: 0.1313 validation: 0.970000\n",
            "Iter-53400 loss: 0.1785 validation: 0.970000\n",
            "Iter-53500 loss: 0.1346 validation: 0.970000\n",
            "Iter-53600 loss: 0.1666 validation: 0.970000\n",
            "Iter-53700 loss: 0.1987 validation: 0.970000\n",
            "Iter-53800 loss: 0.2109 validation: 0.970000\n",
            "Iter-53900 loss: 0.1981 validation: 0.970000\n",
            "Iter-54000 loss: 0.1893 validation: 0.970000\n",
            "Iter-54100 loss: 0.1708 validation: 0.970000\n",
            "Iter-54200 loss: 0.1480 validation: 0.970000\n",
            "Iter-54300 loss: 0.1787 validation: 0.970000\n",
            "Iter-54400 loss: 0.1454 validation: 0.970000\n",
            "Iter-54500 loss: 0.1646 validation: 0.970000\n",
            "Iter-54600 loss: 0.1724 validation: 0.970000\n",
            "Iter-54700 loss: 0.1735 validation: 0.970000\n",
            "Iter-54800 loss: 0.1684 validation: 0.970000\n",
            "Iter-54900 loss: 0.2614 validation: 0.970000\n",
            "Iter-55000 loss: 0.1745 validation: 0.970000\n",
            "Iter-55100 loss: 0.1572 validation: 0.970000\n",
            "Iter-55200 loss: 0.1674 validation: 0.970000\n",
            "Iter-55300 loss: 0.1964 validation: 0.970000\n",
            "Iter-55400 loss: 0.1682 validation: 0.970000\n",
            "Iter-55500 loss: 0.1442 validation: 0.970000\n",
            "Iter-55600 loss: 0.1653 validation: 0.970000\n",
            "Iter-55700 loss: 0.1662 validation: 0.970000\n",
            "Iter-55800 loss: 0.1834 validation: 0.970000\n",
            "Iter-55900 loss: 0.1557 validation: 0.970000\n",
            "Iter-56000 loss: 0.2795 validation: 0.980000\n",
            "Iter-56100 loss: 0.1782 validation: 0.970000\n",
            "Iter-56200 loss: 0.1578 validation: 0.970000\n",
            "Iter-56300 loss: 0.1723 validation: 0.970000\n",
            "Iter-56400 loss: 0.2605 validation: 0.970000\n",
            "Iter-56500 loss: 0.1929 validation: 0.970000\n",
            "Iter-56600 loss: 0.1529 validation: 0.970000\n",
            "Iter-56700 loss: 0.1384 validation: 0.970000\n",
            "Iter-56800 loss: 0.1921 validation: 0.980000\n",
            "Iter-56900 loss: 0.1693 validation: 0.970000\n",
            "Iter-57000 loss: 0.1479 validation: 0.970000\n",
            "Iter-57100 loss: 0.1521 validation: 0.970000\n",
            "Iter-57200 loss: 0.1600 validation: 0.970000\n",
            "Iter-57300 loss: 0.1591 validation: 0.970000\n",
            "Iter-57400 loss: 0.1603 validation: 0.970000\n",
            "Iter-57500 loss: 0.1755 validation: 0.970000\n",
            "Iter-57600 loss: 0.1793 validation: 0.970000\n",
            "Iter-57700 loss: 0.1624 validation: 0.970000\n",
            "Iter-57800 loss: 0.1385 validation: 0.970000\n",
            "Iter-57900 loss: 0.1575 validation: 0.970000\n",
            "Iter-58000 loss: 0.1772 validation: 0.970000\n",
            "Iter-58100 loss: 0.2023 validation: 0.970000\n",
            "Iter-58200 loss: 0.1594 validation: 0.970000\n",
            "Iter-58300 loss: 0.1808 validation: 0.970000\n",
            "Iter-58400 loss: 0.1275 validation: 0.970000\n",
            "Iter-58500 loss: 0.1697 validation: 0.970000\n",
            "Iter-58600 loss: 0.1318 validation: 0.970000\n",
            "Iter-58700 loss: 0.1700 validation: 0.970000\n",
            "Iter-58800 loss: 0.1520 validation: 0.970000\n",
            "Iter-58900 loss: 0.1793 validation: 0.970000\n",
            "Iter-59000 loss: 0.1562 validation: 0.970000\n",
            "Iter-59100 loss: 0.2330 validation: 0.970000\n",
            "Iter-59200 loss: 0.1950 validation: 0.970000\n",
            "Iter-59300 loss: 0.1539 validation: 0.970000\n",
            "Iter-59400 loss: 0.1899 validation: 0.970000\n",
            "Iter-59500 loss: 0.1381 validation: 0.980000\n",
            "Iter-59600 loss: 0.1683 validation: 0.970000\n",
            "Iter-59700 loss: 0.1499 validation: 0.970000\n",
            "Iter-59800 loss: 0.1391 validation: 0.970000\n",
            "Iter-59900 loss: 0.1848 validation: 0.970000\n",
            "Iter-60000 loss: 0.1985 validation: 0.970000\n",
            "Iter-60100 loss: 0.2519 validation: 0.970000\n",
            "Iter-60200 loss: 0.1691 validation: 0.970000\n",
            "Iter-60300 loss: 0.1816 validation: 0.970000\n",
            "Iter-60400 loss: 0.1386 validation: 0.970000\n",
            "Iter-60500 loss: 0.1659 validation: 0.970000\n",
            "Iter-60600 loss: 0.1706 validation: 0.970000\n",
            "Iter-60700 loss: 0.1392 validation: 0.970000\n",
            "Iter-60800 loss: 0.1387 validation: 0.970000\n",
            "Iter-60900 loss: 0.1760 validation: 0.970000\n",
            "Iter-61000 loss: 0.1635 validation: 0.970000\n",
            "Iter-61100 loss: 0.1622 validation: 0.970000\n",
            "Iter-61200 loss: 0.2165 validation: 0.970000\n",
            "Iter-61300 loss: 0.1567 validation: 0.970000\n",
            "Iter-61400 loss: 0.1438 validation: 0.970000\n",
            "Iter-61500 loss: 0.1354 validation: 0.970000\n",
            "Iter-61600 loss: 0.1543 validation: 0.970000\n",
            "Iter-61700 loss: 0.2324 validation: 0.970000\n",
            "Iter-61800 loss: 0.1719 validation: 0.970000\n",
            "Iter-61900 loss: 0.1458 validation: 0.970000\n",
            "Iter-62000 loss: 0.2265 validation: 0.970000\n",
            "Iter-62100 loss: 0.1558 validation: 0.970000\n",
            "Iter-62200 loss: 0.1331 validation: 0.970000\n",
            "Iter-62300 loss: 0.1658 validation: 0.970000\n",
            "Iter-62400 loss: 0.1399 validation: 0.970000\n",
            "Iter-62500 loss: 0.1209 validation: 0.970000\n",
            "Iter-62600 loss: 0.1825 validation: 0.970000\n",
            "Iter-62700 loss: 0.1488 validation: 0.970000\n",
            "Iter-62800 loss: 0.1530 validation: 0.970000\n",
            "Iter-62900 loss: 0.1994 validation: 0.970000\n",
            "Iter-63000 loss: 0.1678 validation: 0.970000\n",
            "Iter-63100 loss: 0.1388 validation: 0.970000\n",
            "Iter-63200 loss: 0.1460 validation: 0.970000\n",
            "Iter-63300 loss: 0.1588 validation: 0.970000\n",
            "Iter-63400 loss: 0.1800 validation: 0.970000\n",
            "Iter-63500 loss: 0.1659 validation: 0.970000\n",
            "Iter-63600 loss: 0.1468 validation: 0.980000\n",
            "Iter-63700 loss: 0.1584 validation: 0.970000\n",
            "Iter-63800 loss: 0.1575 validation: 0.970000\n",
            "Iter-63900 loss: 0.1274 validation: 0.970000\n",
            "Iter-64000 loss: 0.1474 validation: 0.970000\n",
            "Iter-64100 loss: 0.1808 validation: 0.970000\n",
            "Iter-64200 loss: 0.1652 validation: 0.980000\n",
            "Iter-64300 loss: 0.1382 validation: 0.980000\n",
            "Iter-64400 loss: 0.1646 validation: 0.970000\n",
            "Iter-64500 loss: 0.1820 validation: 0.970000\n",
            "Iter-64600 loss: 0.1913 validation: 0.970000\n",
            "Iter-64700 loss: 0.1677 validation: 0.980000\n",
            "Iter-64800 loss: 0.1727 validation: 0.970000\n",
            "Iter-64900 loss: 0.1843 validation: 0.970000\n",
            "Iter-65000 loss: 0.1585 validation: 0.970000\n",
            "Iter-65100 loss: 0.1662 validation: 0.970000\n",
            "Iter-65200 loss: 0.1538 validation: 0.970000\n",
            "Iter-65300 loss: 0.1359 validation: 0.970000\n",
            "Iter-65400 loss: 0.2126 validation: 0.970000\n",
            "Iter-65500 loss: 0.1415 validation: 0.970000\n",
            "Iter-65600 loss: 0.1271 validation: 0.970000\n",
            "Iter-65700 loss: 0.1388 validation: 0.970000\n",
            "Iter-65800 loss: 0.1659 validation: 0.970000\n",
            "Iter-65900 loss: 0.1540 validation: 0.970000\n",
            "Iter-66000 loss: 0.1945 validation: 0.970000\n",
            "Iter-66100 loss: 0.1611 validation: 0.970000\n",
            "Iter-66200 loss: 0.1517 validation: 0.980000\n",
            "Iter-66300 loss: 0.1526 validation: 0.970000\n",
            "Iter-66400 loss: 0.1676 validation: 0.970000\n",
            "Iter-66500 loss: 0.1222 validation: 0.970000\n",
            "Iter-66600 loss: 0.1743 validation: 0.970000\n",
            "Iter-66700 loss: 0.1884 validation: 0.970000\n",
            "Iter-66800 loss: 0.1487 validation: 0.970000\n",
            "Iter-66900 loss: 0.1313 validation: 0.970000\n",
            "Iter-67000 loss: 0.1620 validation: 0.970000\n",
            "Iter-67100 loss: 0.1112 validation: 0.970000\n",
            "Iter-67200 loss: 0.1483 validation: 0.970000\n",
            "Iter-67300 loss: 0.1944 validation: 0.970000\n",
            "Iter-67400 loss: 0.1486 validation: 0.970000\n",
            "Iter-67500 loss: 0.1661 validation: 0.970000\n",
            "Iter-67600 loss: 0.1244 validation: 0.970000\n",
            "Iter-67700 loss: 0.1606 validation: 0.970000\n",
            "Iter-67800 loss: 0.1562 validation: 0.970000\n",
            "Iter-67900 loss: 0.1455 validation: 0.970000\n",
            "Iter-68000 loss: 0.1618 validation: 0.970000\n",
            "Iter-68100 loss: 0.2024 validation: 0.970000\n",
            "Iter-68200 loss: 0.1681 validation: 0.970000\n",
            "Iter-68300 loss: 0.1444 validation: 0.970000\n",
            "Iter-68400 loss: 0.2016 validation: 0.970000\n",
            "Iter-68500 loss: 0.1917 validation: 0.970000\n",
            "Iter-68600 loss: 0.1774 validation: 0.970000\n",
            "Iter-68700 loss: 0.1314 validation: 0.970000\n",
            "Iter-68800 loss: 0.1298 validation: 0.970000\n",
            "Iter-68900 loss: 0.2205 validation: 0.970000\n",
            "Iter-69000 loss: 0.2239 validation: 0.970000\n",
            "Iter-69100 loss: 0.1501 validation: 0.970000\n",
            "Iter-69200 loss: 0.2150 validation: 0.970000\n",
            "Iter-69300 loss: 0.1945 validation: 0.970000\n",
            "Iter-69400 loss: 0.1288 validation: 0.970000\n",
            "Iter-69500 loss: 0.1655 validation: 0.970000\n",
            "Iter-69600 loss: 0.1585 validation: 0.970000\n",
            "Iter-69700 loss: 0.1480 validation: 0.960000\n",
            "Iter-69800 loss: 0.1440 validation: 0.970000\n",
            "Iter-69900 loss: 0.1817 validation: 0.970000\n",
            "Iter-70000 loss: 0.1633 validation: 0.970000\n",
            "Iter-70100 loss: 0.1471 validation: 0.970000\n",
            "Iter-70200 loss: 0.1947 validation: 0.970000\n",
            "Iter-70300 loss: 0.1452 validation: 0.980000\n",
            "Iter-70400 loss: 0.1930 validation: 0.970000\n",
            "Iter-70500 loss: 0.1170 validation: 0.970000\n",
            "Iter-70600 loss: 0.1542 validation: 0.970000\n",
            "Iter-70700 loss: 0.1982 validation: 0.970000\n",
            "Iter-70800 loss: 0.1584 validation: 0.970000\n",
            "Iter-70900 loss: 0.1571 validation: 0.970000\n",
            "Iter-71000 loss: 0.1446 validation: 0.970000\n",
            "Iter-71100 loss: 0.1450 validation: 0.970000\n",
            "Iter-71200 loss: 0.1586 validation: 0.970000\n",
            "Iter-71300 loss: 0.1118 validation: 0.970000\n",
            "Iter-71400 loss: 0.1863 validation: 0.970000\n",
            "Iter-71500 loss: 0.1566 validation: 0.970000\n",
            "Iter-71600 loss: 0.1433 validation: 0.970000\n",
            "Iter-71700 loss: 0.1340 validation: 0.970000\n",
            "Iter-71800 loss: 0.1767 validation: 0.970000\n",
            "Iter-71900 loss: 0.1392 validation: 0.970000\n",
            "Iter-72000 loss: 0.1680 validation: 0.970000\n",
            "Iter-72100 loss: 0.1464 validation: 0.970000\n",
            "Iter-72200 loss: 0.1597 validation: 0.970000\n",
            "Iter-72300 loss: 0.1278 validation: 0.970000\n",
            "Iter-72400 loss: 0.1254 validation: 0.970000\n",
            "Iter-72500 loss: 0.1296 validation: 0.970000\n",
            "Iter-72600 loss: 0.1651 validation: 0.970000\n",
            "Iter-72700 loss: 0.1652 validation: 0.970000\n",
            "Iter-72800 loss: 0.1483 validation: 0.970000\n",
            "Iter-72900 loss: 0.1377 validation: 0.970000\n",
            "Iter-73000 loss: 0.1559 validation: 0.970000\n",
            "Iter-73100 loss: 0.1966 validation: 0.970000\n",
            "Iter-73200 loss: 0.1404 validation: 0.970000\n",
            "Iter-73300 loss: 0.1247 validation: 0.970000\n",
            "Iter-73400 loss: 0.1463 validation: 0.970000\n",
            "Iter-73500 loss: 0.1588 validation: 0.970000\n",
            "Iter-73600 loss: 0.1551 validation: 0.970000\n",
            "Iter-73700 loss: 0.1660 validation: 0.970000\n",
            "Iter-73800 loss: 0.1784 validation: 0.980000\n",
            "Iter-73900 loss: 0.1409 validation: 0.970000\n",
            "Iter-74000 loss: 0.1349 validation: 0.970000\n",
            "Iter-74100 loss: 0.1315 validation: 0.970000\n",
            "Iter-74200 loss: 0.1824 validation: 0.970000\n",
            "Iter-74300 loss: 0.1394 validation: 0.970000\n",
            "Iter-74400 loss: 0.1580 validation: 0.970000\n",
            "Iter-74500 loss: 0.1310 validation: 0.970000\n",
            "Iter-74600 loss: 0.1841 validation: 0.970000\n",
            "Iter-74700 loss: 0.1384 validation: 0.970000\n",
            "Iter-74800 loss: 0.1260 validation: 0.970000\n",
            "Iter-74900 loss: 0.1677 validation: 0.970000\n",
            "Iter-75000 loss: 0.1270 validation: 0.970000\n",
            "Iter-75100 loss: 0.1286 validation: 0.970000\n",
            "Iter-75200 loss: 0.1198 validation: 0.970000\n",
            "Iter-75300 loss: 0.1350 validation: 0.970000\n",
            "Iter-75400 loss: 0.1235 validation: 0.970000\n",
            "Iter-75500 loss: 0.1697 validation: 0.970000\n",
            "Iter-75600 loss: 0.1918 validation: 0.970000\n",
            "Iter-75700 loss: 0.1405 validation: 0.970000\n",
            "Iter-75800 loss: 0.1801 validation: 0.970000\n",
            "Iter-75900 loss: 0.1431 validation: 0.970000\n",
            "Iter-76000 loss: 0.1858 validation: 0.970000\n",
            "Iter-76100 loss: 0.1356 validation: 0.970000\n",
            "Iter-76200 loss: 0.1574 validation: 0.980000\n",
            "Iter-76300 loss: 0.1346 validation: 0.970000\n",
            "Iter-76400 loss: 0.1505 validation: 0.970000\n",
            "Iter-76500 loss: 0.1734 validation: 0.970000\n",
            "Iter-76600 loss: 0.2036 validation: 0.970000\n",
            "Iter-76700 loss: 0.1364 validation: 0.980000\n",
            "Iter-76800 loss: 0.1166 validation: 0.970000\n",
            "Iter-76900 loss: 0.1521 validation: 0.980000\n",
            "Iter-77000 loss: 0.1325 validation: 0.970000\n",
            "Iter-77100 loss: 0.1509 validation: 0.980000\n",
            "Iter-77200 loss: 0.1270 validation: 0.980000\n",
            "Iter-77300 loss: 0.1546 validation: 0.970000\n",
            "Iter-77400 loss: 0.1821 validation: 0.970000\n",
            "Iter-77500 loss: 0.1635 validation: 0.970000\n",
            "Iter-77600 loss: 0.1244 validation: 0.970000\n",
            "Iter-77700 loss: 0.1576 validation: 0.970000\n",
            "Iter-77800 loss: 0.1429 validation: 0.970000\n",
            "Iter-77900 loss: 0.1359 validation: 0.970000\n",
            "Iter-78000 loss: 0.1352 validation: 0.980000\n",
            "Iter-78100 loss: 0.1256 validation: 0.970000\n",
            "Iter-78200 loss: 0.1704 validation: 0.980000\n",
            "Iter-78300 loss: 0.1542 validation: 0.970000\n",
            "Iter-78400 loss: 0.1234 validation: 0.970000\n",
            "Iter-78500 loss: 0.1556 validation: 0.970000\n",
            "Iter-78600 loss: 0.1545 validation: 0.970000\n",
            "Iter-78700 loss: 0.1640 validation: 0.970000\n",
            "Iter-78800 loss: 0.1344 validation: 0.960000\n",
            "Iter-78900 loss: 0.1368 validation: 0.970000\n",
            "Iter-79000 loss: 0.2237 validation: 0.970000\n",
            "Iter-79100 loss: 0.1659 validation: 0.970000\n",
            "Iter-79200 loss: 0.1337 validation: 0.970000\n",
            "Iter-79300 loss: 0.1349 validation: 0.970000\n",
            "Iter-79400 loss: 0.1460 validation: 0.970000\n",
            "Iter-79500 loss: 0.1392 validation: 0.970000\n",
            "Iter-79600 loss: 0.1451 validation: 0.970000\n",
            "Iter-79700 loss: 0.1276 validation: 0.970000\n",
            "Iter-79800 loss: 0.1182 validation: 0.970000\n",
            "Iter-79900 loss: 0.1445 validation: 0.970000\n",
            "Iter-80000 loss: 0.1637 validation: 0.970000\n",
            "Iter-80100 loss: 0.1785 validation: 0.970000\n",
            "Iter-80200 loss: 0.1955 validation: 0.970000\n",
            "Iter-80300 loss: 0.1556 validation: 0.970000\n",
            "Iter-80400 loss: 0.1481 validation: 0.970000\n",
            "Iter-80500 loss: 0.1319 validation: 0.970000\n",
            "Iter-80600 loss: 0.1476 validation: 0.970000\n",
            "Iter-80700 loss: 0.1853 validation: 0.970000\n",
            "Iter-80800 loss: 0.1436 validation: 0.970000\n",
            "Iter-80900 loss: 0.1226 validation: 0.970000\n",
            "Iter-81000 loss: 0.1246 validation: 0.970000\n",
            "Iter-81100 loss: 0.1304 validation: 0.980000\n",
            "Iter-81200 loss: 0.1450 validation: 0.980000\n",
            "Iter-81300 loss: 0.1307 validation: 0.980000\n",
            "Iter-81400 loss: 0.1369 validation: 0.970000\n",
            "Iter-81500 loss: 0.1685 validation: 0.980000\n",
            "Iter-81600 loss: 0.1442 validation: 0.980000\n",
            "Iter-81700 loss: 0.2058 validation: 0.970000\n",
            "Iter-81800 loss: 0.1629 validation: 0.980000\n",
            "Iter-81900 loss: 0.1760 validation: 0.980000\n",
            "Iter-82000 loss: 0.1531 validation: 0.980000\n",
            "Iter-82100 loss: 0.1418 validation: 0.980000\n",
            "Iter-82200 loss: 0.1555 validation: 0.980000\n",
            "Iter-82300 loss: 0.1655 validation: 0.980000\n",
            "Iter-82400 loss: 0.1493 validation: 0.980000\n",
            "Iter-82500 loss: 0.1499 validation: 0.980000\n",
            "Iter-82600 loss: 0.1442 validation: 0.980000\n",
            "Iter-82700 loss: 0.1550 validation: 0.980000\n",
            "Iter-82800 loss: 0.1868 validation: 0.980000\n",
            "Iter-82900 loss: 0.1610 validation: 0.990000\n",
            "Iter-83000 loss: 0.1513 validation: 0.980000\n",
            "Iter-83100 loss: 0.1575 validation: 0.980000\n",
            "Iter-83200 loss: 0.1938 validation: 0.970000\n",
            "Iter-83300 loss: 0.1591 validation: 0.980000\n",
            "Iter-83400 loss: 0.1344 validation: 0.970000\n",
            "Iter-83500 loss: 0.1281 validation: 0.970000\n",
            "Iter-83600 loss: 0.1464 validation: 0.970000\n",
            "Iter-83700 loss: 0.1396 validation: 0.970000\n",
            "Iter-83800 loss: 0.1406 validation: 0.980000\n",
            "Iter-83900 loss: 0.1636 validation: 0.970000\n",
            "Iter-84000 loss: 0.1155 validation: 0.970000\n",
            "Iter-84100 loss: 0.1186 validation: 0.970000\n",
            "Iter-84200 loss: 0.1282 validation: 0.970000\n",
            "Iter-84300 loss: 0.1534 validation: 0.980000\n",
            "Iter-84400 loss: 0.1437 validation: 0.970000\n",
            "Iter-84500 loss: 0.1448 validation: 0.980000\n",
            "Iter-84600 loss: 0.1271 validation: 0.970000\n",
            "Iter-84700 loss: 0.1869 validation: 0.980000\n",
            "Iter-84800 loss: 0.1330 validation: 0.970000\n",
            "Iter-84900 loss: 0.1198 validation: 0.980000\n",
            "Iter-85000 loss: 0.1367 validation: 0.970000\n",
            "Iter-85100 loss: 0.1571 validation: 0.980000\n",
            "Iter-85200 loss: 0.1502 validation: 0.970000\n",
            "Iter-85300 loss: 0.1332 validation: 0.970000\n",
            "Iter-85400 loss: 0.1310 validation: 0.970000\n",
            "Iter-85500 loss: 0.1395 validation: 0.970000\n",
            "Iter-85600 loss: 0.1301 validation: 0.970000\n",
            "Iter-85700 loss: 0.1602 validation: 0.980000\n",
            "Iter-85800 loss: 0.1352 validation: 0.980000\n",
            "Iter-85900 loss: 0.1307 validation: 0.970000\n",
            "Iter-86000 loss: 0.1008 validation: 0.980000\n",
            "Iter-86100 loss: 0.1508 validation: 0.960000\n",
            "Iter-86200 loss: 0.1331 validation: 0.970000\n",
            "Iter-86300 loss: 0.1401 validation: 0.970000\n",
            "Iter-86400 loss: 0.1772 validation: 0.960000\n",
            "Iter-86500 loss: 0.1666 validation: 0.970000\n",
            "Iter-86600 loss: 0.1451 validation: 0.970000\n",
            "Iter-86700 loss: 0.1387 validation: 0.970000\n",
            "Iter-86800 loss: 0.1213 validation: 0.970000\n",
            "Iter-86900 loss: 0.1438 validation: 0.970000\n",
            "Iter-87000 loss: 0.1463 validation: 0.970000\n",
            "Iter-87100 loss: 0.1227 validation: 0.970000\n",
            "Iter-87200 loss: 0.1501 validation: 0.970000\n",
            "Iter-87300 loss: 0.1140 validation: 0.970000\n",
            "Iter-87400 loss: 0.1318 validation: 0.970000\n",
            "Iter-87500 loss: 0.1452 validation: 0.970000\n",
            "Iter-87600 loss: 0.1313 validation: 0.970000\n",
            "Iter-87700 loss: 0.1665 validation: 0.970000\n",
            "Iter-87800 loss: 0.1248 validation: 0.970000\n",
            "Iter-87900 loss: 0.2154 validation: 0.970000\n",
            "Iter-88000 loss: 0.1342 validation: 0.990000\n",
            "Iter-88100 loss: 0.1390 validation: 0.970000\n",
            "Iter-88200 loss: 0.1837 validation: 0.970000\n",
            "Iter-88300 loss: 0.1592 validation: 0.970000\n",
            "Iter-88400 loss: 0.1276 validation: 0.970000\n",
            "Iter-88500 loss: 0.1366 validation: 0.970000\n",
            "Iter-88600 loss: 0.1195 validation: 0.970000\n",
            "Iter-88700 loss: 0.1147 validation: 0.970000\n",
            "Iter-88800 loss: 0.1389 validation: 0.980000\n",
            "Iter-88900 loss: 0.1246 validation: 0.980000\n",
            "Iter-89000 loss: 0.1217 validation: 0.990000\n",
            "Iter-89100 loss: 0.1455 validation: 0.980000\n",
            "Iter-89200 loss: 0.1353 validation: 0.970000\n",
            "Iter-89300 loss: 0.1412 validation: 0.970000\n",
            "Iter-89400 loss: 0.1308 validation: 0.970000\n",
            "Iter-89500 loss: 0.1454 validation: 0.980000\n",
            "Iter-89600 loss: 0.1428 validation: 0.980000\n",
            "Iter-89700 loss: 0.1692 validation: 0.980000\n",
            "Iter-89800 loss: 0.1470 validation: 0.970000\n",
            "Iter-89900 loss: 0.1299 validation: 0.980000\n",
            "Iter-90000 loss: 0.1073 validation: 0.980000\n",
            "Iter-90100 loss: 0.1485 validation: 0.970000\n",
            "Iter-90200 loss: 0.1373 validation: 0.970000\n",
            "Iter-90300 loss: 0.1479 validation: 0.970000\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Z6_fZWL4nLTl"
      },
      "source": [
        "Dropout NN + max-norm constraint + ReLU + 3 layers + 2048 hidden units (50,000 weight updates)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Rh0f9SAgzg-H",
        "outputId": "af8827ab-2d31-4c78-f3e4-8bff4ecec1d0"
      },
      "source": [
        "model = 'ff'\n",
        "n_iter = 50000\n",
        "alpha = 1e-3\n",
        "mb_size = 64\n",
        "n_experiment = 1\n",
        "reg = 1e-5\n",
        "print_after = 100\n",
        "p_dropout = [0.8, 0.5] #if nlayer=n, len(p_dropout) = n-1\n",
        "loss = 'cross_ent'\n",
        "nonlin = 'relu'\n",
        "solver = 'sgd'\n",
        "max_norm = 4\n",
        "nlayer = 3\n",
        "\n",
        "# Define default candidates\n",
        "candidates = np.array([[3, 3.5, 4],[0.795, 0.8, 0.85], [0.495, 0.5, 0.55]]) # first hyper-param as max-norm, second as drop-out rate for input layer, third as drop-out rate for hidden layers\n",
        "\n",
        "run_mnist(2048, model, n_iter, alpha, mb_size, n_experiment, reg, print_after, p_dropout, loss, nonlin, solver, max_norm, x_train, x_validation, x_test, y_train, y_validation, y_test, candidates, nlayer)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Experimenting on sgd\n",
            "\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.5556 validation: 0.830000\n",
            "Iter-200 loss: 1.2308 validation: 0.870000\n",
            "Iter-300 loss: 1.1042 validation: 0.900000\n",
            "Iter-400 loss: 1.0510 validation: 0.910000\n",
            "Iter-500 loss: 1.0171 validation: 0.910000\n",
            "Iter-600 loss: 0.8471 validation: 0.930000\n",
            "Iter-700 loss: 0.9106 validation: 0.940000\n",
            "Iter-800 loss: 0.9128 validation: 0.950000\n",
            "Iter-900 loss: 0.9183 validation: 0.950000\n",
            "Iter-1000 loss: 0.7244 validation: 0.950000\n",
            "Iter-1100 loss: 0.7068 validation: 0.950000\n",
            "Iter-1200 loss: 0.6790 validation: 0.950000\n",
            "Iter-1300 loss: 0.7293 validation: 0.950000\n",
            "Iter-1400 loss: 0.8038 validation: 0.950000\n",
            "Iter-1500 loss: 0.7245 validation: 0.950000\n",
            "Iter-1600 loss: 0.7196 validation: 0.950000\n",
            "Iter-1700 loss: 0.7161 validation: 0.950000\n",
            "Iter-1800 loss: 0.7138 validation: 0.950000\n",
            "Iter-1900 loss: 0.7158 validation: 0.950000\n",
            "Iter-2000 loss: 0.7491 validation: 0.950000\n",
            "Iter-2100 loss: 0.6432 validation: 0.950000\n",
            "Iter-2200 loss: 0.6155 validation: 0.950000\n",
            "Iter-2300 loss: 0.6520 validation: 0.950000\n",
            "Iter-2400 loss: 0.6849 validation: 0.950000\n",
            "Iter-2500 loss: 0.6471 validation: 0.950000\n",
            "Iter-2600 loss: 0.5786 validation: 0.950000\n",
            "Iter-2700 loss: 0.6013 validation: 0.950000\n",
            "Iter-2800 loss: 0.6225 validation: 0.950000\n",
            "Iter-2900 loss: 0.6064 validation: 0.950000\n",
            "Iter-3000 loss: 0.5056 validation: 0.950000\n",
            "Iter-3100 loss: 0.5814 validation: 0.950000\n",
            "Iter-3200 loss: 0.5857 validation: 0.950000\n",
            "Iter-3300 loss: 0.6409 validation: 0.950000\n",
            "Iter-3400 loss: 0.6502 validation: 0.950000\n",
            "Iter-3500 loss: 0.5449 validation: 0.950000\n",
            "Iter-3600 loss: 0.5258 validation: 0.950000\n",
            "Iter-3700 loss: 0.4939 validation: 0.950000\n",
            "Iter-3800 loss: 0.7092 validation: 0.950000\n",
            "Iter-3900 loss: 0.5081 validation: 0.950000\n",
            "Iter-4000 loss: 0.6965 validation: 0.950000\n",
            "Iter-4100 loss: 0.5193 validation: 0.950000\n",
            "Iter-4200 loss: 0.4455 validation: 0.950000\n",
            "Iter-4300 loss: 0.6033 validation: 0.960000\n",
            "Iter-4400 loss: 0.4761 validation: 0.970000\n",
            "Iter-4500 loss: 0.4877 validation: 0.950000\n",
            "Iter-4600 loss: 0.4781 validation: 0.950000\n",
            "Iter-4700 loss: 0.5793 validation: 0.950000\n",
            "Iter-4800 loss: 0.5988 validation: 0.950000\n",
            "Iter-4900 loss: 0.4529 validation: 0.960000\n",
            "Iter-5000 loss: 0.5427 validation: 0.960000\n",
            "Iter-5100 loss: 0.4369 validation: 0.950000\n",
            "Iter-5200 loss: 0.5623 validation: 0.970000\n",
            "Iter-5300 loss: 0.4781 validation: 0.960000\n",
            "Iter-5400 loss: 0.4733 validation: 0.960000\n",
            "Iter-5500 loss: 0.6490 validation: 0.970000\n",
            "Iter-5600 loss: 0.5003 validation: 0.970000\n",
            "Iter-5700 loss: 0.4863 validation: 0.970000\n",
            "Iter-5800 loss: 0.3976 validation: 0.970000\n",
            "Iter-5900 loss: 0.5259 validation: 0.970000\n",
            "Iter-6000 loss: 0.4745 validation: 0.970000\n",
            "Iter-6100 loss: 0.4936 validation: 0.970000\n",
            "Iter-6200 loss: 0.4857 validation: 0.970000\n",
            "Iter-6300 loss: 0.4261 validation: 0.970000\n",
            "Iter-6400 loss: 0.4219 validation: 0.970000\n",
            "Iter-6500 loss: 0.5236 validation: 0.970000\n",
            "Iter-6600 loss: 0.4715 validation: 0.970000\n",
            "Iter-6700 loss: 0.4423 validation: 0.970000\n",
            "Iter-6800 loss: 0.4363 validation: 0.970000\n",
            "Iter-6900 loss: 0.4526 validation: 0.970000\n",
            "Iter-7000 loss: 0.4973 validation: 0.970000\n",
            "Iter-7100 loss: 0.3808 validation: 0.970000\n",
            "Iter-7200 loss: 0.4343 validation: 0.970000\n",
            "Iter-7300 loss: 0.4727 validation: 0.970000\n",
            "Iter-7400 loss: 0.4392 validation: 0.970000\n",
            "Iter-7500 loss: 0.4129 validation: 0.970000\n",
            "Iter-7600 loss: 0.3607 validation: 0.970000\n",
            "Iter-7700 loss: 0.3894 validation: 0.970000\n",
            "Iter-7800 loss: 0.3959 validation: 0.970000\n",
            "Iter-7900 loss: 0.3222 validation: 0.970000\n",
            "Iter-8000 loss: 0.4647 validation: 0.970000\n",
            "Iter-8100 loss: 0.4213 validation: 0.970000\n",
            "Iter-8200 loss: 0.4968 validation: 0.970000\n",
            "Iter-8300 loss: 0.4222 validation: 0.970000\n",
            "Iter-8400 loss: 0.3927 validation: 0.970000\n",
            "Iter-8500 loss: 0.3820 validation: 0.970000\n",
            "Iter-8600 loss: 0.3691 validation: 0.970000\n",
            "Iter-8700 loss: 0.3618 validation: 0.970000\n",
            "Iter-8800 loss: 0.3978 validation: 0.970000\n",
            "Iter-8900 loss: 0.4036 validation: 0.970000\n",
            "Iter-9000 loss: 0.3976 validation: 0.970000\n",
            "Iter-9100 loss: 0.4302 validation: 0.970000\n",
            "Iter-9200 loss: 0.3526 validation: 0.970000\n",
            "Iter-9300 loss: 0.3287 validation: 0.970000\n",
            "Iter-9400 loss: 0.3643 validation: 0.970000\n",
            "Iter-9500 loss: 0.4503 validation: 0.970000\n",
            "Iter-9600 loss: 0.3852 validation: 0.970000\n",
            "Iter-9700 loss: 0.3306 validation: 0.970000\n",
            "Iter-9800 loss: 0.3116 validation: 0.970000\n",
            "Iter-9900 loss: 0.3991 validation: 0.970000\n",
            "Iter-10000 loss: 0.3076 validation: 0.970000\n",
            "Iter-10100 loss: 0.3913 validation: 0.970000\n",
            "Iter-10200 loss: 0.5020 validation: 0.970000\n",
            "Iter-10300 loss: 0.4556 validation: 0.970000\n",
            "Iter-10400 loss: 0.3479 validation: 0.970000\n",
            "Iter-10500 loss: 0.3768 validation: 0.970000\n",
            "Iter-10600 loss: 0.4223 validation: 0.970000\n",
            "Iter-10700 loss: 0.3338 validation: 0.970000\n",
            "Iter-10800 loss: 0.4079 validation: 0.970000\n",
            "Iter-10900 loss: 0.3474 validation: 0.970000\n",
            "Iter-11000 loss: 0.3670 validation: 0.970000\n",
            "Iter-11100 loss: 0.3072 validation: 0.970000\n",
            "Iter-11200 loss: 0.3662 validation: 0.970000\n",
            "Iter-11300 loss: 0.3246 validation: 0.970000\n",
            "Iter-11400 loss: 0.3766 validation: 0.970000\n",
            "Iter-11500 loss: 0.4763 validation: 0.970000\n",
            "Iter-11600 loss: 0.3595 validation: 0.970000\n",
            "Iter-11700 loss: 0.2953 validation: 0.970000\n",
            "Iter-11800 loss: 0.3960 validation: 0.970000\n",
            "Iter-11900 loss: 0.2430 validation: 0.970000\n",
            "Iter-12000 loss: 0.2964 validation: 0.970000\n",
            "Iter-12100 loss: 0.2845 validation: 0.970000\n",
            "Iter-12200 loss: 0.3360 validation: 0.970000\n",
            "Iter-12300 loss: 0.3154 validation: 0.970000\n",
            "Iter-12400 loss: 0.2965 validation: 0.970000\n",
            "Iter-12500 loss: 0.2933 validation: 0.970000\n",
            "Iter-12600 loss: 0.2806 validation: 0.970000\n",
            "Iter-12700 loss: 0.3274 validation: 0.970000\n",
            "Iter-12800 loss: 0.2878 validation: 0.970000\n",
            "Iter-12900 loss: 0.2696 validation: 0.970000\n",
            "Iter-13000 loss: 0.2590 validation: 0.970000\n",
            "Iter-13100 loss: 0.3083 validation: 0.970000\n",
            "Iter-13200 loss: 0.3437 validation: 0.970000\n",
            "Iter-13300 loss: 0.2375 validation: 0.970000\n",
            "Iter-13400 loss: 0.4046 validation: 0.970000\n",
            "Iter-13500 loss: 0.2880 validation: 0.970000\n",
            "Iter-13600 loss: 0.2716 validation: 0.970000\n",
            "Iter-13700 loss: 0.2763 validation: 0.970000\n",
            "Iter-13800 loss: 0.2719 validation: 0.970000\n",
            "Iter-13900 loss: 0.2781 validation: 0.970000\n",
            "Iter-14000 loss: 0.2598 validation: 0.970000\n",
            "Iter-14100 loss: 0.2457 validation: 0.970000\n",
            "Iter-14200 loss: 0.3782 validation: 0.970000\n",
            "Iter-14300 loss: 0.3531 validation: 0.970000\n",
            "Iter-14400 loss: 0.2908 validation: 0.970000\n",
            "Iter-14500 loss: 0.2158 validation: 0.970000\n",
            "Iter-14600 loss: 0.2406 validation: 0.970000\n",
            "Iter-14700 loss: 0.3092 validation: 0.970000\n",
            "Iter-14800 loss: 0.3191 validation: 0.970000\n",
            "Iter-14900 loss: 0.2710 validation: 0.970000\n",
            "Iter-15000 loss: 0.3296 validation: 0.970000\n",
            "Iter-15100 loss: 0.2683 validation: 0.970000\n",
            "Iter-15200 loss: 0.2169 validation: 0.970000\n",
            "Iter-15300 loss: 0.2873 validation: 0.970000\n",
            "Iter-15400 loss: 0.3965 validation: 0.970000\n",
            "Iter-15500 loss: 0.3151 validation: 0.970000\n",
            "Iter-15600 loss: 0.2319 validation: 0.970000\n",
            "Iter-15700 loss: 0.2797 validation: 0.970000\n",
            "Iter-15800 loss: 0.2753 validation: 0.970000\n",
            "Iter-15900 loss: 0.2774 validation: 0.970000\n",
            "Iter-16000 loss: 0.3212 validation: 0.970000\n",
            "Iter-16100 loss: 0.2469 validation: 0.970000\n",
            "Iter-16200 loss: 0.2449 validation: 0.970000\n",
            "Iter-16300 loss: 0.3278 validation: 0.970000\n",
            "Iter-16400 loss: 0.2995 validation: 0.970000\n",
            "Iter-16500 loss: 0.2219 validation: 0.970000\n",
            "Iter-16600 loss: 0.3199 validation: 0.970000\n",
            "Iter-16700 loss: 0.4324 validation: 0.970000\n",
            "Iter-16800 loss: 0.3079 validation: 0.970000\n",
            "Iter-16900 loss: 0.2763 validation: 0.970000\n",
            "Iter-17000 loss: 0.2487 validation: 0.970000\n",
            "Iter-17100 loss: 0.2375 validation: 0.970000\n",
            "Iter-17200 loss: 0.3201 validation: 0.970000\n",
            "Iter-17300 loss: 0.2662 validation: 0.970000\n",
            "Iter-17400 loss: 0.2573 validation: 0.970000\n",
            "Iter-17500 loss: 0.2425 validation: 0.970000\n",
            "Iter-17600 loss: 0.3385 validation: 0.970000\n",
            "Iter-17700 loss: 0.2808 validation: 0.970000\n",
            "Iter-17800 loss: 0.2665 validation: 0.970000\n",
            "Iter-17900 loss: 0.2137 validation: 0.970000\n",
            "Iter-18000 loss: 0.2755 validation: 0.970000\n",
            "Iter-18100 loss: 0.2036 validation: 0.970000\n",
            "Iter-18200 loss: 0.2878 validation: 0.970000\n",
            "Iter-18300 loss: 0.2261 validation: 0.970000\n",
            "Iter-18400 loss: 0.2323 validation: 0.970000\n",
            "Iter-18500 loss: 0.2962 validation: 0.970000\n",
            "Iter-18600 loss: 0.2987 validation: 0.970000\n",
            "Iter-18700 loss: 0.2214 validation: 0.970000\n",
            "Iter-18800 loss: 0.3767 validation: 0.970000\n",
            "Iter-18900 loss: 0.3260 validation: 0.970000\n",
            "Iter-19000 loss: 0.3094 validation: 0.970000\n",
            "Iter-19100 loss: 0.2461 validation: 0.970000\n",
            "Iter-19200 loss: 0.2898 validation: 0.970000\n",
            "Iter-19300 loss: 0.2934 validation: 0.970000\n",
            "Iter-19400 loss: 0.2850 validation: 0.970000\n",
            "Iter-19500 loss: 0.2728 validation: 0.970000\n",
            "Iter-19600 loss: 0.3040 validation: 0.970000\n",
            "Iter-19700 loss: 0.2387 validation: 0.970000\n",
            "Iter-19800 loss: 0.2167 validation: 0.970000\n",
            "Iter-19900 loss: 0.2587 validation: 0.970000\n",
            "Iter-20000 loss: 0.2100 validation: 0.970000\n",
            "Iter-20100 loss: 0.2629 validation: 0.970000\n",
            "Iter-20200 loss: 0.2145 validation: 0.970000\n",
            "Iter-20300 loss: 0.2517 validation: 0.970000\n",
            "Iter-20400 loss: 0.2559 validation: 0.970000\n",
            "Iter-20500 loss: 0.2795 validation: 0.970000\n",
            "Iter-20600 loss: 0.1997 validation: 0.970000\n",
            "Iter-20700 loss: 0.2514 validation: 0.970000\n",
            "Iter-20800 loss: 0.2473 validation: 0.970000\n",
            "Iter-20900 loss: 0.3307 validation: 0.970000\n",
            "Iter-21000 loss: 0.3214 validation: 0.970000\n",
            "Iter-21100 loss: 0.2040 validation: 0.970000\n",
            "Iter-21200 loss: 0.2169 validation: 0.970000\n",
            "Iter-21300 loss: 0.2155 validation: 0.970000\n",
            "Iter-21400 loss: 0.2312 validation: 0.970000\n",
            "Iter-21500 loss: 0.2478 validation: 0.970000\n",
            "Iter-21600 loss: 0.2220 validation: 0.970000\n",
            "Iter-21700 loss: 0.2344 validation: 0.970000\n",
            "Iter-21800 loss: 0.2368 validation: 0.970000\n",
            "Iter-21900 loss: 0.2332 validation: 0.970000\n",
            "Iter-22000 loss: 0.2893 validation: 0.970000\n",
            "Iter-22100 loss: 0.2878 validation: 0.970000\n",
            "Iter-22200 loss: 0.1858 validation: 0.970000\n",
            "Iter-22300 loss: 0.2125 validation: 0.970000\n",
            "Iter-22400 loss: 0.1921 validation: 0.970000\n",
            "Iter-22500 loss: 0.2825 validation: 0.970000\n",
            "Iter-22600 loss: 0.2324 validation: 0.970000\n",
            "Iter-22700 loss: 0.1956 validation: 0.970000\n",
            "Iter-22800 loss: 0.1902 validation: 0.970000\n",
            "Iter-22900 loss: 0.1987 validation: 0.970000\n",
            "Iter-23000 loss: 0.1540 validation: 0.980000\n",
            "Iter-23100 loss: 0.2465 validation: 0.970000\n",
            "Iter-23200 loss: 0.2247 validation: 0.970000\n",
            "Iter-23300 loss: 0.2583 validation: 0.970000\n",
            "Iter-23400 loss: 0.2574 validation: 0.970000\n",
            "Iter-23500 loss: 0.1676 validation: 0.970000\n",
            "Iter-23600 loss: 0.2271 validation: 0.970000\n",
            "Iter-23700 loss: 0.2296 validation: 0.970000\n",
            "Iter-23800 loss: 0.2305 validation: 0.970000\n",
            "Iter-23900 loss: 0.1725 validation: 0.970000\n",
            "Iter-24000 loss: 0.2574 validation: 0.970000\n",
            "Iter-24100 loss: 0.1828 validation: 0.970000\n",
            "Iter-24200 loss: 0.1812 validation: 0.970000\n",
            "Iter-24300 loss: 0.2072 validation: 0.970000\n",
            "Iter-24400 loss: 0.1872 validation: 0.970000\n",
            "Iter-24500 loss: 0.1881 validation: 0.970000\n",
            "Iter-24600 loss: 0.2271 validation: 0.970000\n",
            "Iter-24700 loss: 0.2728 validation: 0.970000\n",
            "Iter-24800 loss: 0.1811 validation: 0.970000\n",
            "Iter-24900 loss: 0.2010 validation: 0.970000\n",
            "Iter-25000 loss: 0.1954 validation: 0.970000\n",
            "Iter-25100 loss: 0.2081 validation: 0.970000\n",
            "Iter-25200 loss: 0.2200 validation: 0.970000\n",
            "Iter-25300 loss: 0.1850 validation: 0.970000\n",
            "Iter-25400 loss: 0.1814 validation: 0.970000\n",
            "Iter-25500 loss: 0.2162 validation: 0.970000\n",
            "Iter-25600 loss: 0.1830 validation: 0.970000\n",
            "Iter-25700 loss: 0.2125 validation: 0.970000\n",
            "Iter-25800 loss: 0.2752 validation: 0.970000\n",
            "Iter-25900 loss: 0.2349 validation: 0.970000\n",
            "Iter-26000 loss: 0.2137 validation: 0.970000\n",
            "Iter-26100 loss: 0.2070 validation: 0.970000\n",
            "Iter-26200 loss: 0.2081 validation: 0.970000\n",
            "Iter-26300 loss: 0.3118 validation: 0.970000\n",
            "Iter-26400 loss: 0.1861 validation: 0.970000\n",
            "Iter-26500 loss: 0.2009 validation: 0.970000\n",
            "Iter-26600 loss: 0.2228 validation: 0.970000\n",
            "Iter-26700 loss: 0.1804 validation: 0.970000\n",
            "Iter-26800 loss: 0.2399 validation: 0.970000\n",
            "Iter-26900 loss: 0.1795 validation: 0.970000\n",
            "Iter-27000 loss: 0.1933 validation: 0.970000\n",
            "Iter-27100 loss: 0.1971 validation: 0.970000\n",
            "Iter-27200 loss: 0.1850 validation: 0.970000\n",
            "Iter-27300 loss: 0.2083 validation: 0.970000\n",
            "Iter-27400 loss: 0.1992 validation: 0.970000\n",
            "Iter-27500 loss: 0.2285 validation: 0.970000\n",
            "Iter-27600 loss: 0.2529 validation: 0.970000\n",
            "Iter-27700 loss: 0.1494 validation: 0.970000\n",
            "Iter-27800 loss: 0.1999 validation: 0.970000\n",
            "Iter-27900 loss: 0.2020 validation: 0.970000\n",
            "Iter-28000 loss: 0.1820 validation: 0.970000\n",
            "Iter-28100 loss: 0.2590 validation: 0.970000\n",
            "Iter-28200 loss: 0.2422 validation: 0.970000\n",
            "Iter-28300 loss: 0.1640 validation: 0.970000\n",
            "Iter-28400 loss: 0.2008 validation: 0.970000\n",
            "Iter-28500 loss: 0.2318 validation: 0.970000\n",
            "Iter-28600 loss: 0.2275 validation: 0.970000\n",
            "Iter-28700 loss: 0.3030 validation: 0.970000\n",
            "Iter-28800 loss: 0.2235 validation: 0.970000\n",
            "Iter-28900 loss: 0.2253 validation: 0.970000\n",
            "Iter-29000 loss: 0.1749 validation: 0.970000\n",
            "Iter-29100 loss: 0.2746 validation: 0.970000\n",
            "Iter-29200 loss: 0.1728 validation: 0.970000\n",
            "Iter-29300 loss: 0.2414 validation: 0.970000\n",
            "Iter-29400 loss: 0.1492 validation: 0.970000\n",
            "Iter-29500 loss: 0.2231 validation: 0.970000\n",
            "Iter-29600 loss: 0.2255 validation: 0.970000\n",
            "Iter-29700 loss: 0.1885 validation: 0.970000\n",
            "Iter-29800 loss: 0.2050 validation: 0.970000\n",
            "Iter-29900 loss: 0.1721 validation: 0.970000\n",
            "Iter-30000 loss: 0.1718 validation: 0.970000\n",
            "Iter-30100 loss: 0.1816 validation: 0.970000\n",
            "Iter-30200 loss: 0.2041 validation: 0.970000\n",
            "Iter-30300 loss: 0.1996 validation: 0.970000\n",
            "Iter-30400 loss: 0.1669 validation: 0.980000\n",
            "Iter-30500 loss: 0.2287 validation: 0.970000\n",
            "Iter-30600 loss: 0.2075 validation: 0.970000\n",
            "Iter-30700 loss: 0.1899 validation: 0.970000\n",
            "Iter-30800 loss: 0.2429 validation: 0.970000\n",
            "Iter-30900 loss: 0.2187 validation: 0.970000\n",
            "Iter-31000 loss: 0.1972 validation: 0.970000\n",
            "Iter-31100 loss: 0.1788 validation: 0.970000\n",
            "Iter-31200 loss: 0.2064 validation: 0.970000\n",
            "Iter-31300 loss: 0.2233 validation: 0.970000\n",
            "Iter-31400 loss: 0.1913 validation: 0.970000\n",
            "Iter-31500 loss: 0.1829 validation: 0.970000\n",
            "Iter-31600 loss: 0.1909 validation: 0.980000\n",
            "Iter-31700 loss: 0.2267 validation: 0.970000\n",
            "Iter-31800 loss: 0.1974 validation: 0.970000\n",
            "Iter-31900 loss: 0.2453 validation: 0.970000\n",
            "Iter-32000 loss: 0.1836 validation: 0.970000\n",
            "Iter-32100 loss: 0.2334 validation: 0.970000\n",
            "Iter-32200 loss: 0.1737 validation: 0.970000\n",
            "Iter-32300 loss: 0.1734 validation: 0.970000\n",
            "Iter-32400 loss: 0.1568 validation: 0.980000\n",
            "Iter-32500 loss: 0.1651 validation: 0.970000\n",
            "Iter-32600 loss: 0.2039 validation: 0.980000\n",
            "Iter-32700 loss: 0.2548 validation: 0.980000\n",
            "Iter-32800 loss: 0.2019 validation: 0.970000\n",
            "Iter-32900 loss: 0.1649 validation: 0.970000\n",
            "Iter-33000 loss: 0.2105 validation: 0.970000\n",
            "Iter-33100 loss: 0.1807 validation: 0.970000\n",
            "Iter-33200 loss: 0.1406 validation: 0.970000\n",
            "Iter-33300 loss: 0.2150 validation: 0.970000\n",
            "Iter-33400 loss: 0.1882 validation: 0.980000\n",
            "Iter-33500 loss: 0.2328 validation: 0.980000\n",
            "Iter-33600 loss: 0.1828 validation: 0.970000\n",
            "Iter-33700 loss: 0.1447 validation: 0.980000\n",
            "Iter-33800 loss: 0.1610 validation: 0.980000\n",
            "Iter-33900 loss: 0.1670 validation: 0.980000\n",
            "Iter-34000 loss: 0.1809 validation: 0.980000\n",
            "Iter-34100 loss: 0.2070 validation: 0.980000\n",
            "Iter-34200 loss: 0.2051 validation: 0.980000\n",
            "Iter-34300 loss: 0.1864 validation: 0.980000\n",
            "Iter-34400 loss: 0.1682 validation: 0.980000\n",
            "Iter-34500 loss: 0.2514 validation: 0.980000\n",
            "Iter-34600 loss: 0.1716 validation: 0.970000\n",
            "Iter-34700 loss: 0.2110 validation: 0.970000\n",
            "Iter-34800 loss: 0.2481 validation: 0.980000\n",
            "Iter-34900 loss: 0.2137 validation: 0.980000\n",
            "Iter-35000 loss: 0.1755 validation: 0.980000\n",
            "Iter-35100 loss: 0.1861 validation: 0.980000\n",
            "Iter-35200 loss: 0.1861 validation: 0.980000\n",
            "Iter-35300 loss: 0.1808 validation: 0.980000\n",
            "Iter-35400 loss: 0.2260 validation: 0.970000\n",
            "Iter-35500 loss: 0.1675 validation: 0.980000\n",
            "Iter-35600 loss: 0.1726 validation: 0.980000\n",
            "Iter-35700 loss: 0.1628 validation: 0.970000\n",
            "Iter-35800 loss: 0.1555 validation: 0.970000\n",
            "Iter-35900 loss: 0.1566 validation: 0.970000\n",
            "Iter-36000 loss: 0.1569 validation: 0.980000\n",
            "Iter-36100 loss: 0.1532 validation: 0.970000\n",
            "Iter-36200 loss: 0.1746 validation: 0.970000\n",
            "Iter-36300 loss: 0.1572 validation: 0.970000\n",
            "Iter-36400 loss: 0.2479 validation: 0.970000\n",
            "Iter-36500 loss: 0.1689 validation: 0.970000\n",
            "Iter-36600 loss: 0.1569 validation: 0.970000\n",
            "Iter-36700 loss: 0.1771 validation: 0.970000\n",
            "Iter-36800 loss: 0.1975 validation: 0.970000\n",
            "Iter-36900 loss: 0.2688 validation: 0.970000\n",
            "Iter-37000 loss: 0.1657 validation: 0.970000\n",
            "Iter-37100 loss: 0.1549 validation: 0.970000\n",
            "Iter-37200 loss: 0.1615 validation: 0.970000\n",
            "Iter-37300 loss: 0.2034 validation: 0.970000\n",
            "Iter-37400 loss: 0.1727 validation: 0.970000\n",
            "Iter-37500 loss: 0.2211 validation: 0.970000\n",
            "Iter-37600 loss: 0.1766 validation: 0.970000\n",
            "Iter-37700 loss: 0.2152 validation: 0.980000\n",
            "Iter-37800 loss: 0.2445 validation: 0.970000\n",
            "Iter-37900 loss: 0.2365 validation: 0.970000\n",
            "Iter-38000 loss: 0.1741 validation: 0.970000\n",
            "Iter-38100 loss: 0.2249 validation: 0.970000\n",
            "Iter-38200 loss: 0.1434 validation: 0.970000\n",
            "Iter-38300 loss: 0.1572 validation: 0.980000\n",
            "Iter-38400 loss: 0.1717 validation: 0.980000\n",
            "Iter-38500 loss: 0.1690 validation: 0.980000\n",
            "Iter-38600 loss: 0.1614 validation: 0.980000\n",
            "Iter-38700 loss: 0.1748 validation: 0.970000\n",
            "Iter-38800 loss: 0.1557 validation: 0.970000\n",
            "Iter-38900 loss: 0.1627 validation: 0.970000\n",
            "Iter-39000 loss: 0.1655 validation: 0.970000\n",
            "Iter-39100 loss: 0.1435 validation: 0.970000\n",
            "Iter-39200 loss: 0.1521 validation: 0.970000\n",
            "Iter-39300 loss: 0.1548 validation: 0.970000\n",
            "Iter-39400 loss: 0.1549 validation: 0.970000\n",
            "Iter-39500 loss: 0.1531 validation: 0.970000\n",
            "Iter-39600 loss: 0.1584 validation: 0.970000\n",
            "Iter-39700 loss: 0.1410 validation: 0.970000\n",
            "Iter-39800 loss: 0.1815 validation: 0.970000\n",
            "Iter-39900 loss: 0.1626 validation: 0.980000\n",
            "Iter-40000 loss: 0.1494 validation: 0.970000\n",
            "Iter-40100 loss: 0.1261 validation: 0.970000\n",
            "Iter-40200 loss: 0.1611 validation: 0.970000\n",
            "Iter-40300 loss: 0.1672 validation: 0.980000\n",
            "Iter-40400 loss: 0.1676 validation: 0.970000\n",
            "Iter-40500 loss: 0.1798 validation: 0.970000\n",
            "Iter-40600 loss: 0.1781 validation: 0.970000\n",
            "Iter-40700 loss: 0.1435 validation: 0.970000\n",
            "Iter-40800 loss: 0.1695 validation: 0.970000\n",
            "Iter-40900 loss: 0.1681 validation: 0.970000\n",
            "Iter-41000 loss: 0.1713 validation: 0.970000\n",
            "Iter-41100 loss: 0.1757 validation: 0.970000\n",
            "Iter-41200 loss: 0.1583 validation: 0.980000\n",
            "Iter-41300 loss: 0.1983 validation: 0.970000\n",
            "Iter-41400 loss: 0.1772 validation: 0.970000\n",
            "Iter-41500 loss: 0.1745 validation: 0.980000\n",
            "Iter-41600 loss: 0.1503 validation: 0.970000\n",
            "Iter-41700 loss: 0.1655 validation: 0.980000\n",
            "Iter-41800 loss: 0.1643 validation: 0.980000\n",
            "Iter-41900 loss: 0.1351 validation: 0.980000\n",
            "Iter-42000 loss: 0.1809 validation: 0.980000\n",
            "Iter-42100 loss: 0.1993 validation: 0.980000\n",
            "Iter-42200 loss: 0.1499 validation: 0.980000\n",
            "Iter-42300 loss: 0.2149 validation: 0.980000\n",
            "Iter-42400 loss: 0.1646 validation: 0.980000\n",
            "Iter-42500 loss: 0.1411 validation: 0.980000\n",
            "Iter-42600 loss: 0.1703 validation: 0.980000\n",
            "Iter-42700 loss: 0.1710 validation: 0.970000\n",
            "Iter-42800 loss: 0.1583 validation: 0.970000\n",
            "Iter-42900 loss: 0.2054 validation: 0.980000\n",
            "Iter-43000 loss: 0.1762 validation: 0.970000\n",
            "Iter-43100 loss: 0.1687 validation: 0.980000\n",
            "Iter-43200 loss: 0.1751 validation: 0.980000\n",
            "Iter-43300 loss: 0.1724 validation: 0.980000\n",
            "Iter-43400 loss: 0.1809 validation: 0.980000\n",
            "Iter-43500 loss: 0.1854 validation: 0.980000\n",
            "Iter-43600 loss: 0.2101 validation: 0.970000\n",
            "Iter-43700 loss: 0.1868 validation: 0.980000\n",
            "Iter-43800 loss: 0.1734 validation: 0.980000\n",
            "Iter-43900 loss: 0.1909 validation: 0.980000\n",
            "Iter-44000 loss: 0.1552 validation: 0.980000\n",
            "Iter-44100 loss: 0.1498 validation: 0.980000\n",
            "Iter-44200 loss: 0.1478 validation: 0.980000\n",
            "Iter-44300 loss: 0.1618 validation: 0.980000\n",
            "Iter-44400 loss: 0.1486 validation: 0.980000\n",
            "Iter-44500 loss: 0.1815 validation: 0.980000\n",
            "Iter-44600 loss: 0.1360 validation: 0.980000\n",
            "Iter-44700 loss: 0.1488 validation: 0.980000\n",
            "Iter-44800 loss: 0.2023 validation: 0.970000\n",
            "Iter-44900 loss: 0.1463 validation: 0.970000\n",
            "Iter-45000 loss: 0.1902 validation: 0.980000\n",
            "Iter-45100 loss: 0.1827 validation: 0.980000\n",
            "Iter-45200 loss: 0.1667 validation: 0.970000\n",
            "Iter-45300 loss: 0.1401 validation: 0.980000\n",
            "Iter-45400 loss: 0.1943 validation: 0.970000\n",
            "Iter-45500 loss: 0.1369 validation: 0.980000\n",
            "Iter-45600 loss: 0.1936 validation: 0.980000\n",
            "Iter-45700 loss: 0.1705 validation: 0.980000\n",
            "Iter-45800 loss: 0.1946 validation: 0.980000\n",
            "Iter-45900 loss: 0.1718 validation: 0.980000\n",
            "Iter-46000 loss: 0.1456 validation: 0.980000\n",
            "Iter-46100 loss: 0.1643 validation: 0.980000\n",
            "Iter-46200 loss: 0.1715 validation: 0.980000\n",
            "Iter-46300 loss: 0.1541 validation: 0.980000\n",
            "Iter-46400 loss: 0.1594 validation: 0.980000\n",
            "Iter-46500 loss: 0.1488 validation: 0.970000\n",
            "Iter-46600 loss: 0.1392 validation: 0.980000\n",
            "Iter-46700 loss: 0.1928 validation: 0.980000\n",
            "Iter-46800 loss: 0.1710 validation: 0.980000\n",
            "Iter-46900 loss: 0.1342 validation: 0.980000\n",
            "Iter-47000 loss: 0.1216 validation: 0.980000\n",
            "Iter-47100 loss: 0.1445 validation: 0.970000\n",
            "Iter-47200 loss: 0.1986 validation: 0.970000\n",
            "Iter-47300 loss: 0.1484 validation: 0.970000\n",
            "Iter-47400 loss: 0.1659 validation: 0.980000\n",
            "Iter-47500 loss: 0.1545 validation: 0.980000\n",
            "Iter-47600 loss: 0.1356 validation: 0.980000\n",
            "Iter-47700 loss: 0.1607 validation: 0.980000\n",
            "Iter-47800 loss: 0.1503 validation: 0.980000\n",
            "Iter-47900 loss: 0.1639 validation: 0.980000\n",
            "Iter-48000 loss: 0.1990 validation: 0.970000\n",
            "Iter-48100 loss: 0.2047 validation: 0.980000\n",
            "Iter-48200 loss: 0.1479 validation: 0.970000\n",
            "Iter-48300 loss: 0.1367 validation: 0.980000\n",
            "Iter-48400 loss: 0.1505 validation: 0.980000\n",
            "Iter-48500 loss: 0.1487 validation: 0.970000\n",
            "Iter-48600 loss: 0.1358 validation: 0.980000\n",
            "Iter-48700 loss: 0.1623 validation: 0.980000\n",
            "Iter-48800 loss: 0.1447 validation: 0.980000\n",
            "Iter-48900 loss: 0.1674 validation: 0.980000\n",
            "Iter-49000 loss: 0.1536 validation: 0.980000\n",
            "Iter-49100 loss: 0.1458 validation: 0.980000\n",
            "Iter-49200 loss: 0.1657 validation: 0.980000\n",
            "Iter-49300 loss: 0.1447 validation: 0.980000\n",
            "Iter-49400 loss: 0.1298 validation: 0.980000\n",
            "Iter-49500 loss: 0.1386 validation: 0.980000\n",
            "Iter-49600 loss: 0.2034 validation: 0.970000\n",
            "Iter-49700 loss: 0.1575 validation: 0.980000\n",
            "Iter-49800 loss: 0.1631 validation: 0.970000\n",
            "Iter-49900 loss: 0.1400 validation: 0.970000\n",
            "Iter-50000 loss: 0.1064 validation: 0.980000\n",
            "\n",
            "Best max_norm:  4.0 best dropout rate for input:  0.8 best dropout rate for hidden layers:  0.55 and result accuracy is:  0.9815\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hgvHx6sYoZq5"
      },
      "source": [
        "Dropout NN + max-norm constraint + ReLU + 3 layers + 1024 hidden units + no reg (50,000 weight updates)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ThQmQ8AUoaKI",
        "outputId": "19ad642a-fd3b-47f7-ddb5-57970cbfcf19"
      },
      "source": [
        "model = 'ff'\n",
        "n_iter = 50000\n",
        "alpha = 1e-3\n",
        "mb_size = 64\n",
        "n_experiment = 1\n",
        "reg = 0\n",
        "print_after = 100\n",
        "p_dropout = [0.8, 0.5] #if nlayer=n, len(p_dropout) = n-1\n",
        "loss = 'cross_ent_noReg'\n",
        "nonlin = 'relu'\n",
        "solver = 'sgd'\n",
        "max_norm = 4\n",
        "nlayer = 3\n",
        "\n",
        "# Define default candidates\n",
        "candidates = np.array([[3, 3.5, 4],[0.795, 0.8, 0.85], [0.495, 0.5, 0.55]]) # first hyper-param as max-norm, second as drop-out rate for input layer, third as drop-out rate for hidden layers\n",
        "\n",
        "run_mnist(1024, model, n_iter, alpha, mb_size, n_experiment, reg, print_after, p_dropout, loss, nonlin, solver, max_norm, x_train, x_validation, x_test, y_train, y_validation, y_test, candidates, nlayer)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Experimenting on sgd\n",
            "\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.6049 validation: 0.820000\n",
            "Iter-200 loss: 1.2869 validation: 0.890000\n",
            "Iter-300 loss: 1.1767 validation: 0.900000\n",
            "Iter-400 loss: 1.1747 validation: 0.900000\n",
            "Iter-500 loss: 1.0189 validation: 0.920000\n",
            "Iter-600 loss: 0.9968 validation: 0.920000\n",
            "Iter-700 loss: 0.8832 validation: 0.950000\n",
            "Iter-800 loss: 0.8489 validation: 0.950000\n",
            "Iter-900 loss: 0.9974 validation: 0.950000\n",
            "Iter-1000 loss: 0.8794 validation: 0.950000\n",
            "Iter-1100 loss: 0.8588 validation: 0.950000\n",
            "Iter-1200 loss: 0.7286 validation: 0.950000\n",
            "Iter-1300 loss: 0.8933 validation: 0.950000\n",
            "Iter-1400 loss: 0.7845 validation: 0.950000\n",
            "Iter-1500 loss: 0.7077 validation: 0.950000\n",
            "Iter-1600 loss: 0.7854 validation: 0.950000\n",
            "Iter-1700 loss: 0.8090 validation: 0.950000\n",
            "Iter-1800 loss: 0.7522 validation: 0.950000\n",
            "Iter-1900 loss: 0.7715 validation: 0.950000\n",
            "Iter-2000 loss: 0.6582 validation: 0.950000\n",
            "Iter-2100 loss: 0.6117 validation: 0.950000\n",
            "Iter-2200 loss: 0.7401 validation: 0.950000\n",
            "Iter-2300 loss: 0.6156 validation: 0.950000\n",
            "Iter-2400 loss: 0.5690 validation: 0.950000\n",
            "Iter-2500 loss: 0.6609 validation: 0.950000\n",
            "Iter-2600 loss: 0.7105 validation: 0.950000\n",
            "Iter-2700 loss: 0.7451 validation: 0.950000\n",
            "Iter-2800 loss: 0.6788 validation: 0.950000\n",
            "Iter-2900 loss: 0.5173 validation: 0.950000\n",
            "Iter-3000 loss: 0.6633 validation: 0.950000\n",
            "Iter-3100 loss: 0.5627 validation: 0.950000\n",
            "Iter-3200 loss: 0.5761 validation: 0.950000\n",
            "Iter-3300 loss: 0.7057 validation: 0.950000\n",
            "Iter-3400 loss: 0.6481 validation: 0.950000\n",
            "Iter-3500 loss: 0.5037 validation: 0.950000\n",
            "Iter-3600 loss: 0.6090 validation: 0.950000\n",
            "Iter-3700 loss: 0.6272 validation: 0.950000\n",
            "Iter-3800 loss: 0.5786 validation: 0.950000\n",
            "Iter-3900 loss: 0.5954 validation: 0.950000\n",
            "Iter-4000 loss: 0.5064 validation: 0.960000\n",
            "Iter-4100 loss: 0.6984 validation: 0.950000\n",
            "Iter-4200 loss: 0.6021 validation: 0.950000\n",
            "Iter-4300 loss: 0.5391 validation: 0.950000\n",
            "Iter-4400 loss: 0.6407 validation: 0.950000\n",
            "Iter-4500 loss: 0.4908 validation: 0.950000\n",
            "Iter-4600 loss: 0.5598 validation: 0.950000\n",
            "Iter-4700 loss: 0.5073 validation: 0.960000\n",
            "Iter-4800 loss: 0.5613 validation: 0.950000\n",
            "Iter-4900 loss: 0.5736 validation: 0.950000\n",
            "Iter-5000 loss: 0.5460 validation: 0.960000\n",
            "Iter-5100 loss: 0.5110 validation: 0.960000\n",
            "Iter-5200 loss: 0.4503 validation: 0.960000\n",
            "Iter-5300 loss: 0.5049 validation: 0.960000\n",
            "Iter-5400 loss: 0.5317 validation: 0.960000\n",
            "Iter-5500 loss: 0.5534 validation: 0.960000\n",
            "Iter-5600 loss: 0.4899 validation: 0.950000\n",
            "Iter-5700 loss: 0.5906 validation: 0.970000\n",
            "Iter-5800 loss: 0.4305 validation: 0.970000\n",
            "Iter-5900 loss: 0.5423 validation: 0.970000\n",
            "Iter-6000 loss: 0.5671 validation: 0.960000\n",
            "Iter-6100 loss: 0.4987 validation: 0.960000\n",
            "Iter-6200 loss: 0.4418 validation: 0.970000\n",
            "Iter-6300 loss: 0.4218 validation: 0.970000\n",
            "Iter-6400 loss: 0.5442 validation: 0.960000\n",
            "Iter-6500 loss: 0.4261 validation: 0.970000\n",
            "Iter-6600 loss: 0.3791 validation: 0.970000\n",
            "Iter-6700 loss: 0.4017 validation: 0.970000\n",
            "Iter-6800 loss: 0.4545 validation: 0.970000\n",
            "Iter-6900 loss: 0.4375 validation: 0.970000\n",
            "Iter-7000 loss: 0.4291 validation: 0.970000\n",
            "Iter-7100 loss: 0.5198 validation: 0.970000\n",
            "Iter-7200 loss: 0.3538 validation: 0.970000\n",
            "Iter-7300 loss: 0.5070 validation: 0.970000\n",
            "Iter-7400 loss: 0.4476 validation: 0.970000\n",
            "Iter-7500 loss: 0.3544 validation: 0.970000\n",
            "Iter-7600 loss: 0.4977 validation: 0.970000\n",
            "Iter-7700 loss: 0.3799 validation: 0.970000\n",
            "Iter-7800 loss: 0.4635 validation: 0.970000\n",
            "Iter-7900 loss: 0.4514 validation: 0.970000\n",
            "Iter-8000 loss: 0.3484 validation: 0.970000\n",
            "Iter-8100 loss: 0.6007 validation: 0.970000\n",
            "Iter-8200 loss: 0.4239 validation: 0.970000\n",
            "Iter-8300 loss: 0.4711 validation: 0.970000\n",
            "Iter-8400 loss: 0.4538 validation: 0.970000\n",
            "Iter-8500 loss: 0.4001 validation: 0.970000\n",
            "Iter-8600 loss: 0.3735 validation: 0.970000\n",
            "Iter-8700 loss: 0.4641 validation: 0.970000\n",
            "Iter-8800 loss: 0.3752 validation: 0.970000\n",
            "Iter-8900 loss: 0.3534 validation: 0.970000\n",
            "Iter-9000 loss: 0.4269 validation: 0.970000\n",
            "Iter-9100 loss: 0.4535 validation: 0.970000\n",
            "Iter-9200 loss: 0.3753 validation: 0.970000\n",
            "Iter-9300 loss: 0.4044 validation: 0.970000\n",
            "Iter-9400 loss: 0.3617 validation: 0.970000\n",
            "Iter-9500 loss: 0.4265 validation: 0.970000\n",
            "Iter-9600 loss: 0.3712 validation: 0.970000\n",
            "Iter-9700 loss: 0.4543 validation: 0.970000\n",
            "Iter-9800 loss: 0.3937 validation: 0.970000\n",
            "Iter-9900 loss: 0.3867 validation: 0.970000\n",
            "Iter-10000 loss: 0.3199 validation: 0.970000\n",
            "Iter-10100 loss: 0.4195 validation: 0.970000\n",
            "Iter-10200 loss: 0.3866 validation: 0.970000\n",
            "Iter-10300 loss: 0.3807 validation: 0.970000\n",
            "Iter-10400 loss: 0.3440 validation: 0.970000\n",
            "Iter-10500 loss: 0.3455 validation: 0.970000\n",
            "Iter-10600 loss: 0.3756 validation: 0.970000\n",
            "Iter-10700 loss: 0.4633 validation: 0.970000\n",
            "Iter-10800 loss: 0.2844 validation: 0.970000\n",
            "Iter-10900 loss: 0.3298 validation: 0.970000\n",
            "Iter-11000 loss: 0.3567 validation: 0.970000\n",
            "Iter-11100 loss: 0.3804 validation: 0.970000\n",
            "Iter-11200 loss: 0.2823 validation: 0.970000\n",
            "Iter-11300 loss: 0.4017 validation: 0.970000\n",
            "Iter-11400 loss: 0.4303 validation: 0.970000\n",
            "Iter-11500 loss: 0.3245 validation: 0.970000\n",
            "Iter-11600 loss: 0.3958 validation: 0.970000\n",
            "Iter-11700 loss: 0.3148 validation: 0.970000\n",
            "Iter-11800 loss: 0.2642 validation: 0.970000\n",
            "Iter-11900 loss: 0.4281 validation: 0.970000\n",
            "Iter-12000 loss: 0.3535 validation: 0.970000\n",
            "Iter-12100 loss: 0.3453 validation: 0.970000\n",
            "Iter-12200 loss: 0.2654 validation: 0.970000\n",
            "Iter-12300 loss: 0.2464 validation: 0.970000\n",
            "Iter-12400 loss: 0.3040 validation: 0.970000\n",
            "Iter-12500 loss: 0.2705 validation: 0.970000\n",
            "Iter-12600 loss: 0.3069 validation: 0.970000\n",
            "Iter-12700 loss: 0.2901 validation: 0.970000\n",
            "Iter-12800 loss: 0.3545 validation: 0.970000\n",
            "Iter-12900 loss: 0.3243 validation: 0.970000\n",
            "Iter-13000 loss: 0.2296 validation: 0.970000\n",
            "Iter-13100 loss: 0.3824 validation: 0.970000\n",
            "Iter-13200 loss: 0.2473 validation: 0.970000\n",
            "Iter-13300 loss: 0.3707 validation: 0.970000\n",
            "Iter-13400 loss: 0.3348 validation: 0.970000\n",
            "Iter-13500 loss: 0.3369 validation: 0.970000\n",
            "Iter-13600 loss: 0.2794 validation: 0.970000\n",
            "Iter-13700 loss: 0.2250 validation: 0.970000\n",
            "Iter-13800 loss: 0.3730 validation: 0.970000\n",
            "Iter-13900 loss: 0.2980 validation: 0.970000\n",
            "Iter-14000 loss: 0.3491 validation: 0.970000\n",
            "Iter-14100 loss: 0.2881 validation: 0.970000\n",
            "Iter-14200 loss: 0.3186 validation: 0.970000\n",
            "Iter-14300 loss: 0.3722 validation: 0.970000\n",
            "Iter-14400 loss: 0.2903 validation: 0.970000\n",
            "Iter-14500 loss: 0.2517 validation: 0.970000\n",
            "Iter-14600 loss: 0.3268 validation: 0.970000\n",
            "Iter-14700 loss: 0.2461 validation: 0.970000\n",
            "Iter-14800 loss: 0.2856 validation: 0.970000\n",
            "Iter-14900 loss: 0.2778 validation: 0.970000\n",
            "Iter-15000 loss: 0.2928 validation: 0.970000\n",
            "Iter-15100 loss: 0.2313 validation: 0.970000\n",
            "Iter-15200 loss: 0.2932 validation: 0.970000\n",
            "Iter-15300 loss: 0.4021 validation: 0.970000\n",
            "Iter-15400 loss: 0.2873 validation: 0.970000\n",
            "Iter-15500 loss: 0.3099 validation: 0.970000\n",
            "Iter-15600 loss: 0.2826 validation: 0.970000\n",
            "Iter-15700 loss: 0.3049 validation: 0.970000\n",
            "Iter-15800 loss: 0.2327 validation: 0.970000\n",
            "Iter-15900 loss: 0.2823 validation: 0.970000\n",
            "Iter-16000 loss: 0.2331 validation: 0.970000\n",
            "Iter-16100 loss: 0.3802 validation: 0.970000\n",
            "Iter-16200 loss: 0.2788 validation: 0.970000\n",
            "Iter-16300 loss: 0.2541 validation: 0.970000\n",
            "Iter-16400 loss: 0.2528 validation: 0.970000\n",
            "Iter-16500 loss: 0.2608 validation: 0.970000\n",
            "Iter-16600 loss: 0.3271 validation: 0.970000\n",
            "Iter-16700 loss: 0.3163 validation: 0.970000\n",
            "Iter-16800 loss: 0.2160 validation: 0.970000\n",
            "Iter-16900 loss: 0.2942 validation: 0.970000\n",
            "Iter-17000 loss: 0.2571 validation: 0.970000\n",
            "Iter-17100 loss: 0.3167 validation: 0.970000\n",
            "Iter-17200 loss: 0.3129 validation: 0.970000\n",
            "Iter-17300 loss: 0.2546 validation: 0.970000\n",
            "Iter-17400 loss: 0.2043 validation: 0.970000\n",
            "Iter-17500 loss: 0.3435 validation: 0.970000\n",
            "Iter-17600 loss: 0.3437 validation: 0.970000\n",
            "Iter-17700 loss: 0.2848 validation: 0.970000\n",
            "Iter-17800 loss: 0.2184 validation: 0.970000\n",
            "Iter-17900 loss: 0.2820 validation: 0.970000\n",
            "Iter-18000 loss: 0.2779 validation: 0.970000\n",
            "Iter-18100 loss: 0.2180 validation: 0.970000\n",
            "Iter-18200 loss: 0.2790 validation: 0.970000\n",
            "Iter-18300 loss: 0.2674 validation: 0.970000\n",
            "Iter-18400 loss: 0.2331 validation: 0.970000\n",
            "Iter-18500 loss: 0.3657 validation: 0.970000\n",
            "Iter-18600 loss: 0.2916 validation: 0.970000\n",
            "Iter-18700 loss: 0.3689 validation: 0.970000\n",
            "Iter-18800 loss: 0.3017 validation: 0.970000\n",
            "Iter-18900 loss: 0.2496 validation: 0.970000\n",
            "Iter-19000 loss: 0.2432 validation: 0.970000\n",
            "Iter-19100 loss: 0.2367 validation: 0.970000\n",
            "Iter-19200 loss: 0.2675 validation: 0.970000\n",
            "Iter-19300 loss: 0.2341 validation: 0.970000\n",
            "Iter-19400 loss: 0.2420 validation: 0.970000\n",
            "Iter-19500 loss: 0.2581 validation: 0.970000\n",
            "Iter-19600 loss: 0.2900 validation: 0.970000\n",
            "Iter-19700 loss: 0.3119 validation: 0.970000\n",
            "Iter-19800 loss: 0.2791 validation: 0.970000\n",
            "Iter-19900 loss: 0.2574 validation: 0.970000\n",
            "Iter-20000 loss: 0.2862 validation: 0.970000\n",
            "Iter-20100 loss: 0.2536 validation: 0.970000\n",
            "Iter-20200 loss: 0.2472 validation: 0.970000\n",
            "Iter-20300 loss: 0.2265 validation: 0.970000\n",
            "Iter-20400 loss: 0.2614 validation: 0.970000\n",
            "Iter-20500 loss: 0.2654 validation: 0.970000\n",
            "Iter-20600 loss: 0.2410 validation: 0.970000\n",
            "Iter-20700 loss: 0.2070 validation: 0.970000\n",
            "Iter-20800 loss: 0.2066 validation: 0.970000\n",
            "Iter-20900 loss: 0.2417 validation: 0.970000\n",
            "Iter-21000 loss: 0.3222 validation: 0.970000\n",
            "Iter-21100 loss: 0.2091 validation: 0.970000\n",
            "Iter-21200 loss: 0.2763 validation: 0.970000\n",
            "Iter-21300 loss: 0.3132 validation: 0.970000\n",
            "Iter-21400 loss: 0.2248 validation: 0.970000\n",
            "Iter-21500 loss: 0.2552 validation: 0.970000\n",
            "Iter-21600 loss: 0.2461 validation: 0.970000\n",
            "Iter-21700 loss: 0.2621 validation: 0.970000\n",
            "Iter-21800 loss: 0.2169 validation: 0.970000\n",
            "Iter-21900 loss: 0.2427 validation: 0.970000\n",
            "Iter-22000 loss: 0.2180 validation: 0.970000\n",
            "Iter-22100 loss: 0.2638 validation: 0.970000\n",
            "Iter-22200 loss: 0.2816 validation: 0.970000\n",
            "Iter-22300 loss: 0.2057 validation: 0.970000\n",
            "Iter-22400 loss: 0.2304 validation: 0.970000\n",
            "Iter-22500 loss: 0.2637 validation: 0.970000\n",
            "Iter-22600 loss: 0.3416 validation: 0.970000\n",
            "Iter-22700 loss: 0.2493 validation: 0.970000\n",
            "Iter-22800 loss: 0.1789 validation: 0.970000\n",
            "Iter-22900 loss: 0.2352 validation: 0.970000\n",
            "Iter-23000 loss: 0.2980 validation: 0.970000\n",
            "Iter-23100 loss: 0.2266 validation: 0.970000\n",
            "Iter-23200 loss: 0.2318 validation: 0.970000\n",
            "Iter-23300 loss: 0.2421 validation: 0.970000\n",
            "Iter-23400 loss: 0.1822 validation: 0.970000\n",
            "Iter-23500 loss: 0.1934 validation: 0.970000\n",
            "Iter-23600 loss: 0.2108 validation: 0.970000\n",
            "Iter-23700 loss: 0.1811 validation: 0.970000\n",
            "Iter-23800 loss: 0.2342 validation: 0.970000\n",
            "Iter-23900 loss: 0.3019 validation: 0.970000\n",
            "Iter-24000 loss: 0.2438 validation: 0.970000\n",
            "Iter-24100 loss: 0.3106 validation: 0.970000\n",
            "Iter-24200 loss: 0.2616 validation: 0.970000\n",
            "Iter-24300 loss: 0.2810 validation: 0.970000\n",
            "Iter-24400 loss: 0.2539 validation: 0.970000\n",
            "Iter-24500 loss: 0.2423 validation: 0.970000\n",
            "Iter-24600 loss: 0.2357 validation: 0.970000\n",
            "Iter-24700 loss: 0.1889 validation: 0.970000\n",
            "Iter-24800 loss: 0.2663 validation: 0.970000\n",
            "Iter-24900 loss: 0.2518 validation: 0.970000\n",
            "Iter-25000 loss: 0.2329 validation: 0.970000\n",
            "Iter-25100 loss: 0.2578 validation: 0.970000\n",
            "Iter-25200 loss: 0.1850 validation: 0.970000\n",
            "Iter-25300 loss: 0.1809 validation: 0.970000\n",
            "Iter-25400 loss: 0.1742 validation: 0.970000\n",
            "Iter-25500 loss: 0.2417 validation: 0.970000\n",
            "Iter-25600 loss: 0.1537 validation: 0.970000\n",
            "Iter-25700 loss: 0.2277 validation: 0.970000\n",
            "Iter-25800 loss: 0.2090 validation: 0.970000\n",
            "Iter-25900 loss: 0.1837 validation: 0.970000\n",
            "Iter-26000 loss: 0.1833 validation: 0.970000\n",
            "Iter-26100 loss: 0.1921 validation: 0.970000\n",
            "Iter-26200 loss: 0.2024 validation: 0.970000\n",
            "Iter-26300 loss: 0.2892 validation: 0.970000\n",
            "Iter-26400 loss: 0.3146 validation: 0.970000\n",
            "Iter-26500 loss: 0.2329 validation: 0.970000\n",
            "Iter-26600 loss: 0.1661 validation: 0.970000\n",
            "Iter-26700 loss: 0.1841 validation: 0.970000\n",
            "Iter-26800 loss: 0.2369 validation: 0.970000\n",
            "Iter-26900 loss: 0.2445 validation: 0.970000\n",
            "Iter-27000 loss: 0.2039 validation: 0.970000\n",
            "Iter-27100 loss: 0.3044 validation: 0.970000\n",
            "Iter-27200 loss: 0.1813 validation: 0.970000\n",
            "Iter-27300 loss: 0.2317 validation: 0.970000\n",
            "Iter-27400 loss: 0.1503 validation: 0.970000\n",
            "Iter-27500 loss: 0.2934 validation: 0.970000\n",
            "Iter-27600 loss: 0.2191 validation: 0.970000\n",
            "Iter-27700 loss: 0.2166 validation: 0.970000\n",
            "Iter-27800 loss: 0.2122 validation: 0.970000\n",
            "Iter-27900 loss: 0.2426 validation: 0.970000\n",
            "Iter-28000 loss: 0.1995 validation: 0.970000\n",
            "Iter-28100 loss: 0.1726 validation: 0.970000\n",
            "Iter-28200 loss: 0.1842 validation: 0.970000\n",
            "Iter-28300 loss: 0.1611 validation: 0.970000\n",
            "Iter-28400 loss: 0.1627 validation: 0.970000\n",
            "Iter-28500 loss: 0.2666 validation: 0.970000\n",
            "Iter-28600 loss: 0.1536 validation: 0.970000\n",
            "Iter-28700 loss: 0.2345 validation: 0.970000\n",
            "Iter-28800 loss: 0.1888 validation: 0.970000\n",
            "Iter-28900 loss: 0.2116 validation: 0.970000\n",
            "Iter-29000 loss: 0.1830 validation: 0.970000\n",
            "Iter-29100 loss: 0.2812 validation: 0.970000\n",
            "Iter-29200 loss: 0.1739 validation: 0.970000\n",
            "Iter-29300 loss: 0.2110 validation: 0.970000\n",
            "Iter-29400 loss: 0.2068 validation: 0.970000\n",
            "Iter-29500 loss: 0.1926 validation: 0.970000\n",
            "Iter-29600 loss: 0.1886 validation: 0.970000\n",
            "Iter-29700 loss: 0.2310 validation: 0.970000\n",
            "Iter-29800 loss: 0.1979 validation: 0.970000\n",
            "Iter-29900 loss: 0.1810 validation: 0.970000\n",
            "Iter-30000 loss: 0.3020 validation: 0.970000\n",
            "Iter-30100 loss: 0.2180 validation: 0.970000\n",
            "Iter-30200 loss: 0.2166 validation: 0.970000\n",
            "Iter-30300 loss: 0.1700 validation: 0.970000\n",
            "Iter-30400 loss: 0.2050 validation: 0.970000\n",
            "Iter-30500 loss: 0.1573 validation: 0.970000\n",
            "Iter-30600 loss: 0.2118 validation: 0.970000\n",
            "Iter-30700 loss: 0.1714 validation: 0.970000\n",
            "Iter-30800 loss: 0.2257 validation: 0.970000\n",
            "Iter-30900 loss: 0.2233 validation: 0.970000\n",
            "Iter-31000 loss: 0.2411 validation: 0.970000\n",
            "Iter-31100 loss: 0.1630 validation: 0.970000\n",
            "Iter-31200 loss: 0.2119 validation: 0.970000\n",
            "Iter-31300 loss: 0.1996 validation: 0.970000\n",
            "Iter-31400 loss: 0.2046 validation: 0.970000\n",
            "Iter-31500 loss: 0.1736 validation: 0.970000\n",
            "Iter-31600 loss: 0.1644 validation: 0.970000\n",
            "Iter-31700 loss: 0.2667 validation: 0.970000\n",
            "Iter-31800 loss: 0.1800 validation: 0.970000\n",
            "Iter-31900 loss: 0.1728 validation: 0.970000\n",
            "Iter-32000 loss: 0.2012 validation: 0.970000\n",
            "Iter-32100 loss: 0.1429 validation: 0.970000\n",
            "Iter-32200 loss: 0.1914 validation: 0.970000\n",
            "Iter-32300 loss: 0.1433 validation: 0.970000\n",
            "Iter-32400 loss: 0.1693 validation: 0.970000\n",
            "Iter-32500 loss: 0.1722 validation: 0.970000\n",
            "Iter-32600 loss: 0.1757 validation: 0.970000\n",
            "Iter-32700 loss: 0.2269 validation: 0.970000\n",
            "Iter-32800 loss: 0.1868 validation: 0.970000\n",
            "Iter-32900 loss: 0.2001 validation: 0.970000\n",
            "Iter-33000 loss: 0.1833 validation: 0.970000\n",
            "Iter-33100 loss: 0.1934 validation: 0.970000\n",
            "Iter-33200 loss: 0.1524 validation: 0.970000\n",
            "Iter-33300 loss: 0.1897 validation: 0.970000\n",
            "Iter-33400 loss: 0.1650 validation: 0.970000\n",
            "Iter-33500 loss: 0.1423 validation: 0.970000\n",
            "Iter-33600 loss: 0.1938 validation: 0.970000\n",
            "Iter-33700 loss: 0.1578 validation: 0.970000\n",
            "Iter-33800 loss: 0.2113 validation: 0.970000\n",
            "Iter-33900 loss: 0.1515 validation: 0.970000\n",
            "Iter-34000 loss: 0.1571 validation: 0.970000\n",
            "Iter-34100 loss: 0.2777 validation: 0.970000\n",
            "Iter-34200 loss: 0.2086 validation: 0.970000\n",
            "Iter-34300 loss: 0.1747 validation: 0.970000\n",
            "Iter-34400 loss: 0.1429 validation: 0.970000\n",
            "Iter-34500 loss: 0.2378 validation: 0.970000\n",
            "Iter-34600 loss: 0.1633 validation: 0.970000\n",
            "Iter-34700 loss: 0.1783 validation: 0.970000\n",
            "Iter-34800 loss: 0.3255 validation: 0.970000\n",
            "Iter-34900 loss: 0.1563 validation: 0.970000\n",
            "Iter-35000 loss: 0.1934 validation: 0.970000\n",
            "Iter-35100 loss: 0.1678 validation: 0.970000\n",
            "Iter-35200 loss: 0.2009 validation: 0.970000\n",
            "Iter-35300 loss: 0.1826 validation: 0.970000\n",
            "Iter-35400 loss: 0.2158 validation: 0.970000\n",
            "Iter-35500 loss: 0.1702 validation: 0.970000\n",
            "Iter-35600 loss: 0.2315 validation: 0.970000\n",
            "Iter-35700 loss: 0.1697 validation: 0.970000\n",
            "Iter-35800 loss: 0.1940 validation: 0.970000\n",
            "Iter-35900 loss: 0.1613 validation: 0.970000\n",
            "Iter-36000 loss: 0.1701 validation: 0.970000\n",
            "Iter-36100 loss: 0.1387 validation: 0.970000\n",
            "Iter-36200 loss: 0.1749 validation: 0.970000\n",
            "Iter-36300 loss: 0.2215 validation: 0.970000\n",
            "Iter-36400 loss: 0.1791 validation: 0.970000\n",
            "Iter-36500 loss: 0.2131 validation: 0.970000\n",
            "Iter-36600 loss: 0.1662 validation: 0.970000\n",
            "Iter-36700 loss: 0.1917 validation: 0.970000\n",
            "Iter-36800 loss: 0.1735 validation: 0.970000\n",
            "Iter-36900 loss: 0.2068 validation: 0.970000\n",
            "Iter-37000 loss: 0.1868 validation: 0.970000\n",
            "Iter-37100 loss: 0.1779 validation: 0.970000\n",
            "Iter-37200 loss: 0.1609 validation: 0.970000\n",
            "Iter-37300 loss: 0.1473 validation: 0.970000\n",
            "Iter-37400 loss: 0.2311 validation: 0.970000\n",
            "Iter-37500 loss: 0.1946 validation: 0.970000\n",
            "Iter-37600 loss: 0.2339 validation: 0.970000\n",
            "Iter-37700 loss: 0.1921 validation: 0.970000\n",
            "Iter-37800 loss: 0.2384 validation: 0.970000\n",
            "Iter-37900 loss: 0.1507 validation: 0.970000\n",
            "Iter-38000 loss: 0.1699 validation: 0.970000\n",
            "Iter-38100 loss: 0.2039 validation: 0.970000\n",
            "Iter-38200 loss: 0.1625 validation: 0.970000\n",
            "Iter-38300 loss: 0.1511 validation: 0.970000\n",
            "Iter-38400 loss: 0.1467 validation: 0.970000\n",
            "Iter-38500 loss: 0.2482 validation: 0.970000\n",
            "Iter-38600 loss: 0.1874 validation: 0.970000\n",
            "Iter-38700 loss: 0.1722 validation: 0.970000\n",
            "Iter-38800 loss: 0.1285 validation: 0.970000\n",
            "Iter-38900 loss: 0.1424 validation: 0.970000\n",
            "Iter-39000 loss: 0.2002 validation: 0.970000\n",
            "Iter-39100 loss: 0.1572 validation: 0.970000\n",
            "Iter-39200 loss: 0.1311 validation: 0.970000\n",
            "Iter-39300 loss: 0.2233 validation: 0.970000\n",
            "Iter-39400 loss: 0.1382 validation: 0.970000\n",
            "Iter-39500 loss: 0.1571 validation: 0.970000\n",
            "Iter-39600 loss: 0.1627 validation: 0.970000\n",
            "Iter-39700 loss: 0.1816 validation: 0.970000\n",
            "Iter-39800 loss: 0.2044 validation: 0.970000\n",
            "Iter-39900 loss: 0.1553 validation: 0.970000\n",
            "Iter-40000 loss: 0.2521 validation: 0.970000\n",
            "Iter-40100 loss: 0.1425 validation: 0.970000\n",
            "Iter-40200 loss: 0.2044 validation: 0.970000\n",
            "Iter-40300 loss: 0.1866 validation: 0.970000\n",
            "Iter-40400 loss: 0.1964 validation: 0.970000\n",
            "Iter-40500 loss: 0.1560 validation: 0.970000\n",
            "Iter-40600 loss: 0.2241 validation: 0.970000\n",
            "Iter-40700 loss: 0.2503 validation: 0.970000\n",
            "Iter-40800 loss: 0.1629 validation: 0.970000\n",
            "Iter-40900 loss: 0.1888 validation: 0.970000\n",
            "Iter-41000 loss: 0.1507 validation: 0.970000\n",
            "Iter-41100 loss: 0.1755 validation: 0.970000\n",
            "Iter-41200 loss: 0.2212 validation: 0.970000\n",
            "Iter-41300 loss: 0.1827 validation: 0.970000\n",
            "Iter-41400 loss: 0.1945 validation: 0.970000\n",
            "Iter-41500 loss: 0.1910 validation: 0.970000\n",
            "Iter-41600 loss: 0.1625 validation: 0.970000\n",
            "Iter-41700 loss: 0.2457 validation: 0.970000\n",
            "Iter-41800 loss: 0.1554 validation: 0.970000\n",
            "Iter-41900 loss: 0.2008 validation: 0.970000\n",
            "Iter-42000 loss: 0.2315 validation: 0.970000\n",
            "Iter-42100 loss: 0.1552 validation: 0.970000\n",
            "Iter-42200 loss: 0.1952 validation: 0.970000\n",
            "Iter-42300 loss: 0.1819 validation: 0.970000\n",
            "Iter-42400 loss: 0.1836 validation: 0.970000\n",
            "Iter-42500 loss: 0.1720 validation: 0.970000\n",
            "Iter-42600 loss: 0.1900 validation: 0.970000\n",
            "Iter-42700 loss: 0.1502 validation: 0.980000\n",
            "Iter-42800 loss: 0.1711 validation: 0.970000\n",
            "Iter-42900 loss: 0.2170 validation: 0.970000\n",
            "Iter-43000 loss: 0.2004 validation: 0.970000\n",
            "Iter-43100 loss: 0.1729 validation: 0.970000\n",
            "Iter-43200 loss: 0.2246 validation: 0.970000\n",
            "Iter-43300 loss: 0.1586 validation: 0.970000\n",
            "Iter-43400 loss: 0.1906 validation: 0.970000\n",
            "Iter-43500 loss: 0.1575 validation: 0.970000\n",
            "Iter-43600 loss: 0.1417 validation: 0.970000\n",
            "Iter-43700 loss: 0.1588 validation: 0.970000\n",
            "Iter-43800 loss: 0.1398 validation: 0.970000\n",
            "Iter-43900 loss: 0.1135 validation: 0.970000\n",
            "Iter-44000 loss: 0.1734 validation: 0.970000\n",
            "Iter-44100 loss: 0.2228 validation: 0.970000\n",
            "Iter-44200 loss: 0.2033 validation: 0.970000\n",
            "Iter-44300 loss: 0.1865 validation: 0.970000\n",
            "Iter-44400 loss: 0.1588 validation: 0.970000\n",
            "Iter-44500 loss: 0.2112 validation: 0.970000\n",
            "Iter-44600 loss: 0.1561 validation: 0.970000\n",
            "Iter-44700 loss: 0.1889 validation: 0.970000\n",
            "Iter-44800 loss: 0.1489 validation: 0.970000\n",
            "Iter-44900 loss: 0.1146 validation: 0.970000\n",
            "Iter-45000 loss: 0.1619 validation: 0.970000\n",
            "Iter-45100 loss: 0.1362 validation: 0.980000\n",
            "Iter-45200 loss: 0.1350 validation: 0.970000\n",
            "Iter-45300 loss: 0.1522 validation: 0.970000\n",
            "Iter-45400 loss: 0.1756 validation: 0.980000\n",
            "Iter-45500 loss: 0.1658 validation: 0.970000\n",
            "Iter-45600 loss: 0.1598 validation: 0.970000\n",
            "Iter-45700 loss: 0.1599 validation: 0.980000\n",
            "Iter-45800 loss: 0.1764 validation: 0.970000\n",
            "Iter-45900 loss: 0.1898 validation: 0.970000\n",
            "Iter-46000 loss: 0.1273 validation: 0.970000\n",
            "Iter-46100 loss: 0.1749 validation: 0.970000\n",
            "Iter-46200 loss: 0.1187 validation: 0.970000\n",
            "Iter-46300 loss: 0.1694 validation: 0.970000\n",
            "Iter-46400 loss: 0.1727 validation: 0.970000\n",
            "Iter-46500 loss: 0.1696 validation: 0.970000\n",
            "Iter-46600 loss: 0.2271 validation: 0.970000\n",
            "Iter-46700 loss: 0.1271 validation: 0.970000\n",
            "Iter-46800 loss: 0.1559 validation: 0.970000\n",
            "Iter-46900 loss: 0.1477 validation: 0.970000\n",
            "Iter-47000 loss: 0.1527 validation: 0.970000\n",
            "Iter-47100 loss: 0.1503 validation: 0.970000\n",
            "Iter-47200 loss: 0.1657 validation: 0.980000\n",
            "Iter-47300 loss: 0.1417 validation: 0.970000\n",
            "Iter-47400 loss: 0.1368 validation: 0.970000\n",
            "Iter-47500 loss: 0.1586 validation: 0.970000\n",
            "Iter-47600 loss: 0.1660 validation: 0.970000\n",
            "Iter-47700 loss: 0.2264 validation: 0.980000\n",
            "Iter-47800 loss: 0.1653 validation: 0.970000\n",
            "Iter-47900 loss: 0.1778 validation: 0.970000\n",
            "Iter-48000 loss: 0.2042 validation: 0.970000\n",
            "Iter-48100 loss: 0.1375 validation: 0.970000\n",
            "Iter-48200 loss: 0.1089 validation: 0.970000\n",
            "Iter-48300 loss: 0.2097 validation: 0.970000\n",
            "Iter-48400 loss: 0.1369 validation: 0.970000\n",
            "Iter-48500 loss: 0.1448 validation: 0.970000\n",
            "Iter-48600 loss: 0.1660 validation: 0.970000\n",
            "Iter-48700 loss: 0.1488 validation: 0.970000\n",
            "Iter-48800 loss: 0.1880 validation: 0.970000\n",
            "Iter-48900 loss: 0.1733 validation: 0.970000\n",
            "Iter-49000 loss: 0.1861 validation: 0.970000\n",
            "Iter-49100 loss: 0.1393 validation: 0.970000\n",
            "Iter-49200 loss: 0.1482 validation: 0.970000\n",
            "Iter-49300 loss: 0.1709 validation: 0.970000\n",
            "Iter-49400 loss: 0.1455 validation: 0.980000\n",
            "Iter-49500 loss: 0.1370 validation: 0.970000\n",
            "Iter-49600 loss: 0.1338 validation: 0.970000\n",
            "Iter-49700 loss: 0.1343 validation: 0.970000\n",
            "Iter-49800 loss: 0.1648 validation: 0.970000\n",
            "Iter-49900 loss: 0.2065 validation: 0.970000\n",
            "Iter-50000 loss: 0.2586 validation: 0.970000\n",
            "\n",
            "Best max_norm:  4.0 best dropout rate for input:  0.85 best dropout rate for hidden layers:  0.5 and result accuracy is:  0.9805\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yCr9t2gf4BIZ"
      },
      "source": [
        "Dropout NN + ReLU + 3 layers + 1024 hidden units + data augmentation (50,000 weight updates)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uZcjt1RD4B5I",
        "outputId": "8c6d7287-f511-4e0a-d0fa-f1a346ccdfe1"
      },
      "source": [
        "model = 'ff'\n",
        "n_iter = 500\n",
        "alpha = 1e-3\n",
        "mb_size = 64\n",
        "n_experiment = 1\n",
        "reg = 1e-5\n",
        "print_after = 100\n",
        "p_dropout = [0.8, 0.5] #if nlayer=n, len(p_dropout) = n-1\n",
        "loss = 'cross_ent'\n",
        "nonlin = 'relu'\n",
        "solver = 'sgd'\n",
        "max_norm = float('inf')\n",
        "nlayer = 3\n",
        "\n",
        "# Define default candidates\n",
        "candidates = np.array([[3, 3.5, 4], [0.795, 0.8, 0.85],[0.495, 0.5, 0.55]]) # first hyper-param as max-norm, second as drop-out rate for input layer, third as drop-out rate for hidden layers\n",
        "\n",
        "run_mnist(1024, model, n_iter, alpha, mb_size, n_experiment, reg, print_after, p_dropout, loss, nonlin, solver, max_norm, x_train_aug, x_validation_aug, x_test, y_train_aug, y_validation_aug, y_test, candidates, nlayer)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Experimenting on sgd\n",
            "\n",
            "Experiment-1\n",
            "Iter-100 loss: 2.2325 validation: 0.510000\n",
            "Iter-200 loss: 1.8849 validation: 0.690000\n",
            "Iter-300 loss: 1.6153 validation: 0.760000\n",
            "Iter-400 loss: 1.2131 validation: 0.800000\n",
            "Iter-500 loss: 1.0173 validation: 0.820000\n",
            "Iter-600 loss: 1.2380 validation: 0.840000\n",
            "Iter-700 loss: 1.1989 validation: 0.870000\n",
            "Iter-800 loss: 0.6675 validation: 0.870000\n",
            "Iter-900 loss: 0.7968 validation: 0.890000\n",
            "Iter-1000 loss: 0.6454 validation: 0.890000\n",
            "Iter-1100 loss: 0.6967 validation: 0.890000\n",
            "Iter-1200 loss: 0.6438 validation: 0.890000\n",
            "Iter-1300 loss: 0.9898 validation: 0.890000\n",
            "Iter-1400 loss: 0.7700 validation: 0.890000\n",
            "Iter-1500 loss: 0.4295 validation: 0.900000\n",
            "Iter-1600 loss: 0.6714 validation: 0.890000\n",
            "Iter-1700 loss: 0.6316 validation: 0.890000\n",
            "Iter-1800 loss: 0.8387 validation: 0.910000\n",
            "Iter-1900 loss: 0.5955 validation: 0.910000\n",
            "Iter-2000 loss: 0.6797 validation: 0.900000\n",
            "Iter-2100 loss: 0.5798 validation: 0.920000\n",
            "Iter-2200 loss: 0.8080 validation: 0.920000\n",
            "Iter-2300 loss: 0.5923 validation: 0.920000\n",
            "Iter-2400 loss: 0.6612 validation: 0.920000\n",
            "Iter-2500 loss: 0.5259 validation: 0.920000\n",
            "Iter-2600 loss: 0.6366 validation: 0.930000\n",
            "Iter-2700 loss: 0.5002 validation: 0.930000\n",
            "Iter-2800 loss: 0.3455 validation: 0.930000\n",
            "Iter-2900 loss: 0.7664 validation: 0.930000\n",
            "Iter-3000 loss: 0.5438 validation: 0.920000\n",
            "Iter-3100 loss: 0.7280 validation: 0.930000\n",
            "Iter-3200 loss: 0.7081 validation: 0.920000\n",
            "Iter-3300 loss: 0.6880 validation: 0.930000\n",
            "Iter-3400 loss: 0.7388 validation: 0.930000\n",
            "Iter-3500 loss: 0.4458 validation: 0.940000\n",
            "Iter-3600 loss: 0.3824 validation: 0.940000\n",
            "Iter-3700 loss: 0.9326 validation: 0.940000\n",
            "Iter-3800 loss: 0.6307 validation: 0.940000\n",
            "Iter-3900 loss: 0.4657 validation: 0.940000\n",
            "Iter-4000 loss: 0.5561 validation: 0.940000\n",
            "Iter-4100 loss: 0.5230 validation: 0.940000\n",
            "Iter-4200 loss: 0.3738 validation: 0.940000\n",
            "Iter-4300 loss: 0.4331 validation: 0.940000\n",
            "Iter-4400 loss: 0.4506 validation: 0.940000\n",
            "Iter-4500 loss: 0.5795 validation: 0.940000\n",
            "Iter-4600 loss: 0.3600 validation: 0.940000\n",
            "Iter-4700 loss: 0.4650 validation: 0.940000\n",
            "Iter-4800 loss: 0.4469 validation: 0.940000\n",
            "Iter-4900 loss: 0.6133 validation: 0.940000\n",
            "Iter-5000 loss: 0.4912 validation: 0.940000\n",
            "Iter-5100 loss: 0.5955 validation: 0.940000\n",
            "Iter-5200 loss: 0.3987 validation: 0.940000\n",
            "Iter-5300 loss: 0.4189 validation: 0.940000\n",
            "Iter-5400 loss: 0.3587 validation: 0.940000\n",
            "Iter-5500 loss: 0.3917 validation: 0.940000\n",
            "Iter-5600 loss: 0.4530 validation: 0.940000\n",
            "Iter-5700 loss: 0.5444 validation: 0.940000\n",
            "Iter-5800 loss: 0.6177 validation: 0.940000\n",
            "Iter-5900 loss: 0.4532 validation: 0.940000\n",
            "Iter-6000 loss: 0.4344 validation: 0.940000\n",
            "Iter-6100 loss: 0.7601 validation: 0.940000\n",
            "Iter-6200 loss: 0.4764 validation: 0.940000\n",
            "Iter-6300 loss: 0.3570 validation: 0.940000\n",
            "Iter-6400 loss: 0.6080 validation: 0.940000\n",
            "Iter-6500 loss: 0.2652 validation: 0.940000\n",
            "Iter-6600 loss: 0.3466 validation: 0.940000\n",
            "Iter-6700 loss: 0.5855 validation: 0.940000\n",
            "Iter-6800 loss: 0.4684 validation: 0.940000\n",
            "Iter-6900 loss: 0.2751 validation: 0.940000\n",
            "Iter-7000 loss: 0.3849 validation: 0.940000\n",
            "Iter-7100 loss: 0.2761 validation: 0.940000\n",
            "Iter-7200 loss: 0.3157 validation: 0.950000\n",
            "Iter-7300 loss: 0.3975 validation: 0.940000\n",
            "Iter-7400 loss: 0.4260 validation: 0.940000\n",
            "Iter-7500 loss: 0.4716 validation: 0.940000\n",
            "Iter-7600 loss: 0.5159 validation: 0.940000\n",
            "Iter-7700 loss: 0.3634 validation: 0.940000\n",
            "Iter-7800 loss: 0.4601 validation: 0.940000\n",
            "Iter-7900 loss: 0.2859 validation: 0.940000\n",
            "Iter-8000 loss: 0.2195 validation: 0.940000\n",
            "Iter-8100 loss: 0.3087 validation: 0.940000\n",
            "Iter-8200 loss: 0.3657 validation: 0.950000\n",
            "Iter-8300 loss: 0.4358 validation: 0.940000\n",
            "Iter-8400 loss: 0.3825 validation: 0.950000\n",
            "Iter-8500 loss: 0.3353 validation: 0.940000\n",
            "Iter-8600 loss: 0.3503 validation: 0.940000\n",
            "Iter-8700 loss: 0.2378 validation: 0.950000\n",
            "Iter-8800 loss: 0.3594 validation: 0.940000\n",
            "Iter-8900 loss: 0.2888 validation: 0.940000\n",
            "Iter-9000 loss: 0.2894 validation: 0.950000\n",
            "Iter-9100 loss: 0.4663 validation: 0.950000\n",
            "Iter-9200 loss: 0.2423 validation: 0.950000\n",
            "Iter-9300 loss: 0.3923 validation: 0.950000\n",
            "Iter-9400 loss: 0.3409 validation: 0.950000\n",
            "Iter-9500 loss: 0.3813 validation: 0.950000\n",
            "Iter-9600 loss: 0.4617 validation: 0.950000\n",
            "Iter-9700 loss: 0.2502 validation: 0.950000\n",
            "Iter-9800 loss: 0.3889 validation: 0.940000\n",
            "Iter-9900 loss: 0.1849 validation: 0.950000\n",
            "Iter-10000 loss: 0.2762 validation: 0.950000\n",
            "Iter-10100 loss: 0.4700 validation: 0.960000\n",
            "Iter-10200 loss: 0.3244 validation: 0.950000\n",
            "Iter-10300 loss: 0.2639 validation: 0.940000\n",
            "Iter-10400 loss: 0.3633 validation: 0.950000\n",
            "Iter-10500 loss: 0.3646 validation: 0.950000\n",
            "Iter-10600 loss: 0.4429 validation: 0.950000\n",
            "Iter-10700 loss: 0.2931 validation: 0.950000\n",
            "Iter-10800 loss: 0.4316 validation: 0.960000\n",
            "Iter-10900 loss: 0.2839 validation: 0.950000\n",
            "Iter-11000 loss: 0.3082 validation: 0.960000\n",
            "Iter-11100 loss: 0.3674 validation: 0.960000\n",
            "Iter-11200 loss: 0.5874 validation: 0.960000\n",
            "Iter-11300 loss: 0.1801 validation: 0.960000\n",
            "Iter-11400 loss: 0.3691 validation: 0.960000\n",
            "Iter-11500 loss: 0.4438 validation: 0.960000\n",
            "Iter-11600 loss: 0.3303 validation: 0.950000\n",
            "Iter-11700 loss: 0.3668 validation: 0.960000\n",
            "Iter-11800 loss: 0.4280 validation: 0.960000\n",
            "Iter-11900 loss: 0.3391 validation: 0.960000\n",
            "Iter-12000 loss: 0.3319 validation: 0.950000\n",
            "Iter-12100 loss: 0.3482 validation: 0.960000\n",
            "Iter-12200 loss: 0.3238 validation: 0.950000\n",
            "Iter-12300 loss: 0.5538 validation: 0.960000\n",
            "Iter-12400 loss: 0.2977 validation: 0.960000\n",
            "Iter-12500 loss: 0.2442 validation: 0.960000\n",
            "Iter-12600 loss: 0.5323 validation: 0.960000\n",
            "Iter-12700 loss: 0.4781 validation: 0.960000\n",
            "Iter-12800 loss: 0.2651 validation: 0.950000\n",
            "Iter-12900 loss: 0.3456 validation: 0.960000\n",
            "Iter-13000 loss: 0.5040 validation: 0.960000\n",
            "Iter-13100 loss: 0.3780 validation: 0.960000\n",
            "Iter-13200 loss: 0.3434 validation: 0.960000\n",
            "Iter-13300 loss: 0.2754 validation: 0.960000\n",
            "Iter-13400 loss: 0.2931 validation: 0.960000\n",
            "Iter-13500 loss: 0.1319 validation: 0.960000\n",
            "Iter-13600 loss: 0.3287 validation: 0.960000\n",
            "Iter-13700 loss: 0.3995 validation: 0.960000\n",
            "Iter-13800 loss: 0.3405 validation: 0.960000\n",
            "Iter-13900 loss: 0.2326 validation: 0.960000\n",
            "Iter-14000 loss: 0.2515 validation: 0.960000\n",
            "Iter-14100 loss: 0.3335 validation: 0.960000\n",
            "Iter-14200 loss: 0.2923 validation: 0.960000\n",
            "Iter-14300 loss: 0.2998 validation: 0.960000\n",
            "Iter-14400 loss: 0.2989 validation: 0.950000\n",
            "Iter-14500 loss: 0.3635 validation: 0.960000\n",
            "Iter-14600 loss: 0.3955 validation: 0.960000\n",
            "Iter-14700 loss: 0.2552 validation: 0.950000\n",
            "Iter-14800 loss: 0.1672 validation: 0.960000\n",
            "Iter-14900 loss: 0.2775 validation: 0.960000\n",
            "Iter-15000 loss: 0.1895 validation: 0.960000\n",
            "Iter-15100 loss: 0.2254 validation: 0.960000\n",
            "Iter-15200 loss: 0.4628 validation: 0.960000\n",
            "Iter-15300 loss: 0.4472 validation: 0.960000\n",
            "Iter-15400 loss: 0.2237 validation: 0.960000\n",
            "Iter-15500 loss: 0.3018 validation: 0.960000\n",
            "Iter-15600 loss: 0.1864 validation: 0.960000\n",
            "Iter-15700 loss: 0.4205 validation: 0.960000\n",
            "Iter-15800 loss: 0.3081 validation: 0.960000\n",
            "Iter-15900 loss: 0.2795 validation: 0.960000\n",
            "Iter-16000 loss: 0.1455 validation: 0.960000\n",
            "Iter-16100 loss: 0.4442 validation: 0.960000\n",
            "Iter-16200 loss: 0.2934 validation: 0.960000\n",
            "Iter-16300 loss: 0.5075 validation: 0.970000\n",
            "Iter-16400 loss: 0.3016 validation: 0.960000\n",
            "Iter-16500 loss: 0.2655 validation: 0.960000\n",
            "Iter-16600 loss: 0.2642 validation: 0.970000\n",
            "Iter-16700 loss: 0.2548 validation: 0.970000\n",
            "Iter-16800 loss: 0.3990 validation: 0.970000\n",
            "Iter-16900 loss: 0.2944 validation: 0.970000\n",
            "Iter-17000 loss: 0.2667 validation: 0.970000\n",
            "Iter-17100 loss: 0.4433 validation: 0.960000\n",
            "Iter-17200 loss: 0.1488 validation: 0.970000\n",
            "Iter-17300 loss: 0.2001 validation: 0.970000\n",
            "Iter-17400 loss: 0.2290 validation: 0.960000\n",
            "Iter-17500 loss: 0.2593 validation: 0.970000\n",
            "Iter-17600 loss: 0.1687 validation: 0.970000\n",
            "Iter-17700 loss: 0.6144 validation: 0.970000\n",
            "Iter-17800 loss: 0.3279 validation: 0.970000\n",
            "Iter-17900 loss: 0.3059 validation: 0.960000\n",
            "Iter-18000 loss: 0.2783 validation: 0.970000\n",
            "Iter-18100 loss: 0.3772 validation: 0.970000\n",
            "Iter-18200 loss: 0.2130 validation: 0.970000\n",
            "Iter-18300 loss: 0.3051 validation: 0.970000\n",
            "Iter-18400 loss: 0.3061 validation: 0.970000\n",
            "Iter-18500 loss: 0.1639 validation: 0.970000\n",
            "Iter-18600 loss: 0.2551 validation: 0.970000\n",
            "Iter-18700 loss: 0.2656 validation: 0.970000\n",
            "Iter-18800 loss: 0.2582 validation: 0.970000\n",
            "Iter-18900 loss: 0.2187 validation: 0.970000\n",
            "Iter-19000 loss: 0.2466 validation: 0.970000\n",
            "Iter-19100 loss: 0.5209 validation: 0.970000\n",
            "Iter-19200 loss: 0.2661 validation: 0.970000\n",
            "Iter-19300 loss: 0.4590 validation: 0.970000\n",
            "Iter-19400 loss: 0.2070 validation: 0.970000\n",
            "Iter-19500 loss: 0.1789 validation: 0.970000\n",
            "Iter-19600 loss: 0.2563 validation: 0.970000\n",
            "Iter-19700 loss: 0.3437 validation: 0.970000\n",
            "Iter-19800 loss: 0.5630 validation: 0.970000\n",
            "Iter-19900 loss: 0.4366 validation: 0.970000\n",
            "Iter-20000 loss: 0.0893 validation: 0.970000\n",
            "Iter-20100 loss: 0.2717 validation: 0.970000\n",
            "Iter-20200 loss: 0.2925 validation: 0.970000\n",
            "Iter-20300 loss: 0.2124 validation: 0.970000\n",
            "Iter-20400 loss: 0.3835 validation: 0.970000\n",
            "Iter-20500 loss: 0.4399 validation: 0.970000\n",
            "Iter-20600 loss: 0.2135 validation: 0.970000\n",
            "Iter-20700 loss: 0.2029 validation: 0.970000\n",
            "Iter-20800 loss: 0.3528 validation: 0.970000\n",
            "Iter-20900 loss: 0.3132 validation: 0.970000\n",
            "Iter-21000 loss: 0.2989 validation: 0.970000\n",
            "Iter-21100 loss: 0.3965 validation: 0.960000\n",
            "Iter-21200 loss: 0.3712 validation: 0.970000\n",
            "Iter-21300 loss: 0.1566 validation: 0.970000\n",
            "Iter-21400 loss: 0.2636 validation: 0.970000\n",
            "Iter-21500 loss: 0.4924 validation: 0.970000\n",
            "Iter-21600 loss: 0.4355 validation: 0.970000\n",
            "Iter-21700 loss: 0.2417 validation: 0.970000\n",
            "Iter-21800 loss: 0.4281 validation: 0.970000\n",
            "Iter-21900 loss: 0.2425 validation: 0.970000\n",
            "Iter-22000 loss: 0.4195 validation: 0.970000\n",
            "Iter-22100 loss: 0.2676 validation: 0.970000\n",
            "Iter-22200 loss: 0.2578 validation: 0.970000\n",
            "Iter-22300 loss: 0.4077 validation: 0.970000\n",
            "Iter-22400 loss: 0.3358 validation: 0.970000\n",
            "Iter-22500 loss: 0.2129 validation: 0.970000\n",
            "Iter-22600 loss: 0.3157 validation: 0.970000\n",
            "Iter-22700 loss: 0.1982 validation: 0.970000\n",
            "Iter-22800 loss: 0.1756 validation: 0.970000\n",
            "Iter-22900 loss: 0.2678 validation: 0.970000\n",
            "Iter-23000 loss: 0.1797 validation: 0.970000\n",
            "Iter-23100 loss: 0.2970 validation: 0.970000\n",
            "Iter-23200 loss: 0.1624 validation: 0.970000\n",
            "Iter-23300 loss: 0.3374 validation: 0.970000\n",
            "Iter-23400 loss: 0.2528 validation: 0.970000\n",
            "Iter-23500 loss: 0.4027 validation: 0.970000\n",
            "Iter-23600 loss: 0.2242 validation: 0.970000\n",
            "Iter-23700 loss: 0.2612 validation: 0.970000\n",
            "Iter-23800 loss: 0.2165 validation: 0.970000\n",
            "Iter-23900 loss: 0.4657 validation: 0.970000\n",
            "Iter-24000 loss: 0.2278 validation: 0.970000\n",
            "Iter-24100 loss: 0.1820 validation: 0.970000\n",
            "Iter-24200 loss: 0.2450 validation: 0.970000\n",
            "Iter-24300 loss: 0.3048 validation: 0.970000\n",
            "Iter-24400 loss: 0.4264 validation: 0.970000\n",
            "Iter-24500 loss: 0.1322 validation: 0.970000\n",
            "Iter-24600 loss: 0.2040 validation: 0.970000\n",
            "Iter-24700 loss: 0.2739 validation: 0.970000\n",
            "Iter-24800 loss: 0.4507 validation: 0.970000\n",
            "Iter-24900 loss: 0.2796 validation: 0.970000\n",
            "Iter-25000 loss: 0.2459 validation: 0.970000\n",
            "Iter-25100 loss: 0.2592 validation: 0.970000\n",
            "Iter-25200 loss: 0.1377 validation: 0.970000\n",
            "Iter-25300 loss: 0.2382 validation: 0.970000\n",
            "Iter-25400 loss: 0.4128 validation: 0.970000\n",
            "Iter-25500 loss: 0.1776 validation: 0.970000\n",
            "Iter-25600 loss: 0.2118 validation: 0.970000\n",
            "Iter-25700 loss: 0.1489 validation: 0.970000\n",
            "Iter-25800 loss: 0.3321 validation: 0.970000\n",
            "Iter-25900 loss: 0.3662 validation: 0.970000\n",
            "Iter-26000 loss: 0.3345 validation: 0.970000\n",
            "Iter-26100 loss: 0.2117 validation: 0.970000\n",
            "Iter-26200 loss: 0.5135 validation: 0.970000\n",
            "Iter-26300 loss: 0.3490 validation: 0.970000\n",
            "Iter-26400 loss: 0.3526 validation: 0.970000\n",
            "Iter-26500 loss: 0.3259 validation: 0.970000\n",
            "Iter-26600 loss: 0.4107 validation: 0.970000\n",
            "Iter-26700 loss: 0.2186 validation: 0.970000\n",
            "Iter-26800 loss: 0.1406 validation: 0.970000\n",
            "Iter-26900 loss: 0.3090 validation: 0.970000\n",
            "Iter-27000 loss: 0.3070 validation: 0.970000\n",
            "Iter-27100 loss: 0.2089 validation: 0.970000\n",
            "Iter-27200 loss: 0.2286 validation: 0.970000\n",
            "Iter-27300 loss: 0.2127 validation: 0.970000\n",
            "Iter-27400 loss: 0.2774 validation: 0.970000\n",
            "Iter-27500 loss: 0.2742 validation: 0.970000\n",
            "Iter-27600 loss: 0.1893 validation: 0.970000\n",
            "Iter-27700 loss: 0.1934 validation: 0.970000\n",
            "Iter-27800 loss: 0.3872 validation: 0.970000\n",
            "Iter-27900 loss: 0.2983 validation: 0.970000\n",
            "Iter-28000 loss: 0.2193 validation: 0.970000\n",
            "Iter-28100 loss: 0.1942 validation: 0.970000\n",
            "Iter-28200 loss: 0.1276 validation: 0.970000\n",
            "Iter-28300 loss: 0.2986 validation: 0.970000\n",
            "Iter-28400 loss: 0.1939 validation: 0.970000\n",
            "Iter-28500 loss: 0.2185 validation: 0.970000\n",
            "Iter-28600 loss: 0.3137 validation: 0.970000\n",
            "Iter-28700 loss: 0.4310 validation: 0.970000\n",
            "Iter-28800 loss: 0.3771 validation: 0.970000\n",
            "Iter-28900 loss: 0.2128 validation: 0.970000\n",
            "Iter-29000 loss: 0.2767 validation: 0.970000\n",
            "Iter-29100 loss: 0.1740 validation: 0.970000\n",
            "Iter-29200 loss: 0.2407 validation: 0.970000\n",
            "Iter-29300 loss: 0.1896 validation: 0.970000\n",
            "Iter-29400 loss: 0.4469 validation: 0.970000\n",
            "Iter-29500 loss: 0.3262 validation: 0.970000\n",
            "Iter-29600 loss: 0.2860 validation: 0.970000\n",
            "Iter-29700 loss: 0.3249 validation: 0.970000\n",
            "Iter-29800 loss: 0.1617 validation: 0.970000\n",
            "Iter-29900 loss: 0.2271 validation: 0.970000\n",
            "Iter-30000 loss: 0.3698 validation: 0.970000\n",
            "Iter-30100 loss: 0.2220 validation: 0.970000\n",
            "Iter-30200 loss: 0.3479 validation: 0.970000\n",
            "Iter-30300 loss: 0.3821 validation: 0.970000\n",
            "Iter-30400 loss: 0.1436 validation: 0.970000\n",
            "Iter-30500 loss: 0.2752 validation: 0.970000\n",
            "Iter-30600 loss: 0.5474 validation: 0.970000\n",
            "Iter-30700 loss: 0.1338 validation: 0.970000\n",
            "Iter-30800 loss: 0.3779 validation: 0.970000\n",
            "Iter-30900 loss: 0.2055 validation: 0.970000\n",
            "Iter-31000 loss: 0.3171 validation: 0.970000\n",
            "Iter-31100 loss: 0.1052 validation: 0.970000\n",
            "Iter-31200 loss: 0.2651 validation: 0.970000\n",
            "Iter-31300 loss: 0.3611 validation: 0.970000\n",
            "Iter-31400 loss: 0.1279 validation: 0.970000\n",
            "Iter-31500 loss: 0.2067 validation: 0.970000\n",
            "Iter-31600 loss: 0.2284 validation: 0.970000\n",
            "Iter-31700 loss: 0.1697 validation: 0.970000\n",
            "Iter-31800 loss: 0.3019 validation: 0.970000\n",
            "Iter-31900 loss: 0.2589 validation: 0.970000\n",
            "Iter-32000 loss: 0.2658 validation: 0.970000\n",
            "Iter-32100 loss: 0.2246 validation: 0.970000\n",
            "Iter-32200 loss: 0.2337 validation: 0.970000\n",
            "Iter-32300 loss: 0.1975 validation: 0.970000\n",
            "Iter-32400 loss: 0.2392 validation: 0.970000\n",
            "Iter-32500 loss: 0.2262 validation: 0.970000\n",
            "Iter-32600 loss: 0.1863 validation: 0.970000\n",
            "Iter-32700 loss: 0.3671 validation: 0.970000\n",
            "Iter-32800 loss: 0.2413 validation: 0.970000\n",
            "Iter-32900 loss: 0.2488 validation: 0.970000\n",
            "Iter-33000 loss: 0.2252 validation: 0.970000\n",
            "Iter-33100 loss: 0.3362 validation: 0.970000\n",
            "Iter-33200 loss: 0.2541 validation: 0.970000\n",
            "Iter-33300 loss: 0.3156 validation: 0.970000\n",
            "Iter-33400 loss: 0.2056 validation: 0.970000\n",
            "Iter-33500 loss: 0.1850 validation: 0.970000\n",
            "Iter-33600 loss: 0.3672 validation: 0.970000\n",
            "Iter-33700 loss: 0.3832 validation: 0.970000\n",
            "Iter-33800 loss: 0.1384 validation: 0.970000\n",
            "Iter-33900 loss: 0.1952 validation: 0.970000\n",
            "Iter-34000 loss: 0.1519 validation: 0.970000\n",
            "Iter-34100 loss: 0.1694 validation: 0.970000\n",
            "Iter-34200 loss: 0.1378 validation: 0.970000\n",
            "Iter-34300 loss: 0.3363 validation: 0.970000\n",
            "Iter-34400 loss: 0.2690 validation: 0.970000\n",
            "Iter-34500 loss: 0.1051 validation: 0.970000\n",
            "Iter-34600 loss: 0.2991 validation: 0.970000\n",
            "Iter-34700 loss: 0.1421 validation: 0.970000\n",
            "Iter-34800 loss: 0.1276 validation: 0.970000\n",
            "Iter-34900 loss: 0.2733 validation: 0.970000\n",
            "Iter-35000 loss: 0.2564 validation: 0.970000\n",
            "Iter-35100 loss: 0.3532 validation: 0.970000\n",
            "Iter-35200 loss: 0.1535 validation: 0.970000\n",
            "Iter-35300 loss: 0.3360 validation: 0.970000\n",
            "Iter-35400 loss: 0.2196 validation: 0.970000\n",
            "Iter-35500 loss: 0.1594 validation: 0.970000\n",
            "Iter-35600 loss: 0.2695 validation: 0.970000\n",
            "Iter-35700 loss: 0.5514 validation: 0.970000\n",
            "Iter-35800 loss: 0.1048 validation: 0.970000\n",
            "Iter-35900 loss: 0.2420 validation: 0.970000\n",
            "Iter-36000 loss: 0.3157 validation: 0.970000\n",
            "Iter-36100 loss: 0.3417 validation: 0.970000\n",
            "Iter-36200 loss: 0.3589 validation: 0.970000\n",
            "Iter-36300 loss: 0.2699 validation: 0.970000\n",
            "Iter-36400 loss: 0.3624 validation: 0.970000\n",
            "Iter-36500 loss: 0.3523 validation: 0.970000\n",
            "Iter-36600 loss: 0.1867 validation: 0.970000\n",
            "Iter-36700 loss: 0.1904 validation: 0.970000\n",
            "Iter-36800 loss: 0.1989 validation: 0.970000\n",
            "Iter-36900 loss: 0.3453 validation: 0.970000\n",
            "Iter-37000 loss: 0.1915 validation: 0.970000\n",
            "Iter-37100 loss: 0.2175 validation: 0.970000\n",
            "Iter-37200 loss: 0.1716 validation: 0.970000\n",
            "Iter-37300 loss: 0.1601 validation: 0.970000\n",
            "Iter-37400 loss: 0.1764 validation: 0.970000\n",
            "Iter-37500 loss: 0.1299 validation: 0.970000\n",
            "Iter-37600 loss: 0.3241 validation: 0.970000\n",
            "Iter-37700 loss: 0.3187 validation: 0.970000\n",
            "Iter-37800 loss: 0.2245 validation: 0.970000\n",
            "Iter-37900 loss: 0.1843 validation: 0.970000\n",
            "Iter-38000 loss: 0.4005 validation: 0.970000\n",
            "Iter-38100 loss: 0.2952 validation: 0.970000\n",
            "Iter-38200 loss: 0.1778 validation: 0.970000\n",
            "Iter-38300 loss: 0.2575 validation: 0.970000\n",
            "Iter-38400 loss: 0.1845 validation: 0.970000\n",
            "Iter-38500 loss: 0.1369 validation: 0.970000\n",
            "Iter-38600 loss: 0.1793 validation: 0.970000\n",
            "Iter-38700 loss: 0.2196 validation: 0.970000\n",
            "Iter-38800 loss: 0.2159 validation: 0.970000\n",
            "Iter-38900 loss: 0.2858 validation: 0.970000\n",
            "Iter-39000 loss: 0.2550 validation: 0.970000\n",
            "Iter-39100 loss: 0.2196 validation: 0.970000\n",
            "Iter-39200 loss: 0.2285 validation: 0.970000\n",
            "Iter-39300 loss: 0.3270 validation: 0.970000\n",
            "Iter-39400 loss: 0.1402 validation: 0.970000\n",
            "Iter-39500 loss: 0.1123 validation: 0.970000\n",
            "Iter-39600 loss: 0.2557 validation: 0.970000\n",
            "Iter-39700 loss: 0.1947 validation: 0.970000\n",
            "Iter-39800 loss: 0.2225 validation: 0.970000\n",
            "Iter-39900 loss: 0.1928 validation: 0.970000\n",
            "Iter-40000 loss: 0.2109 validation: 0.970000\n",
            "Iter-40100 loss: 0.3582 validation: 0.970000\n",
            "Iter-40200 loss: 0.1015 validation: 0.970000\n",
            "Iter-40300 loss: 0.3035 validation: 0.970000\n",
            "Iter-40400 loss: 0.2082 validation: 0.970000\n",
            "Iter-40500 loss: 0.1743 validation: 0.970000\n",
            "Iter-40600 loss: 0.1597 validation: 0.970000\n",
            "Iter-40700 loss: 0.1888 validation: 0.970000\n",
            "Iter-40800 loss: 0.1848 validation: 0.970000\n",
            "Iter-40900 loss: 0.1884 validation: 0.970000\n",
            "Iter-41000 loss: 0.2338 validation: 0.970000\n",
            "Iter-41100 loss: 0.2804 validation: 0.970000\n",
            "Iter-41200 loss: 0.2836 validation: 0.970000\n",
            "Iter-41300 loss: 0.1419 validation: 0.970000\n",
            "Iter-41400 loss: 0.1921 validation: 0.970000\n",
            "Iter-41500 loss: 0.1550 validation: 0.970000\n",
            "Iter-41600 loss: 0.1420 validation: 0.970000\n",
            "Iter-41700 loss: 0.1237 validation: 0.970000\n",
            "Iter-41800 loss: 0.1768 validation: 0.970000\n",
            "Iter-41900 loss: 0.1676 validation: 0.970000\n",
            "Iter-42000 loss: 0.2190 validation: 0.970000\n",
            "Iter-42100 loss: 0.2203 validation: 0.970000\n",
            "Iter-42200 loss: 0.1316 validation: 0.970000\n",
            "Iter-42300 loss: 0.2305 validation: 0.970000\n",
            "Iter-42400 loss: 0.3226 validation: 0.970000\n",
            "Iter-42500 loss: 0.3698 validation: 0.970000\n",
            "Iter-42600 loss: 0.1952 validation: 0.970000\n",
            "Iter-42700 loss: 0.2730 validation: 0.970000\n",
            "Iter-42800 loss: 0.3029 validation: 0.970000\n",
            "Iter-42900 loss: 0.1587 validation: 0.970000\n",
            "Iter-43000 loss: 0.2117 validation: 0.970000\n",
            "Iter-43100 loss: 0.2156 validation: 0.970000\n",
            "Iter-43200 loss: 0.3766 validation: 0.970000\n",
            "Iter-43300 loss: 0.0896 validation: 0.970000\n",
            "Iter-43400 loss: 0.1761 validation: 0.970000\n",
            "Iter-43500 loss: 0.1891 validation: 0.970000\n",
            "Iter-43600 loss: 0.3709 validation: 0.970000\n",
            "Iter-43700 loss: 0.1562 validation: 0.970000\n",
            "Iter-43800 loss: 0.2461 validation: 0.970000\n",
            "Iter-43900 loss: 0.1188 validation: 0.970000\n",
            "Iter-44000 loss: 0.3221 validation: 0.970000\n",
            "Iter-44100 loss: 0.2712 validation: 0.970000\n",
            "Iter-44200 loss: 0.4266 validation: 0.970000\n",
            "Iter-44300 loss: 0.1233 validation: 0.970000\n",
            "Iter-44400 loss: 0.1975 validation: 0.970000\n",
            "Iter-44500 loss: 0.3161 validation: 0.970000\n",
            "Iter-44600 loss: 0.2188 validation: 0.970000\n",
            "Iter-44700 loss: 0.2257 validation: 0.970000\n",
            "Iter-44800 loss: 0.2128 validation: 0.970000\n",
            "Iter-44900 loss: 0.4118 validation: 0.970000\n",
            "Iter-45000 loss: 0.2495 validation: 0.970000\n",
            "Iter-45100 loss: 0.2743 validation: 0.970000\n",
            "Iter-45200 loss: 0.1011 validation: 0.970000\n",
            "Iter-45300 loss: 0.1597 validation: 0.970000\n",
            "Iter-45400 loss: 0.2335 validation: 0.970000\n",
            "Iter-45500 loss: 0.2510 validation: 0.970000\n",
            "Iter-45600 loss: 0.1436 validation: 0.970000\n",
            "Iter-45700 loss: 0.2280 validation: 0.970000\n",
            "Iter-45800 loss: 0.2344 validation: 0.970000\n",
            "Iter-45900 loss: 0.4377 validation: 0.970000\n",
            "Iter-46000 loss: 0.2080 validation: 0.970000\n",
            "Iter-46100 loss: 0.2921 validation: 0.970000\n",
            "Iter-46200 loss: 0.1174 validation: 0.970000\n",
            "Iter-46300 loss: 0.2418 validation: 0.970000\n",
            "Iter-46400 loss: 0.2386 validation: 0.970000\n",
            "Iter-46500 loss: 0.2462 validation: 0.970000\n",
            "Iter-46600 loss: 0.1483 validation: 0.970000\n",
            "Iter-46700 loss: 0.1972 validation: 0.970000\n",
            "Iter-46800 loss: 0.2175 validation: 0.970000\n",
            "Iter-46900 loss: 0.1107 validation: 0.970000\n",
            "Iter-47000 loss: 0.1083 validation: 0.970000\n",
            "Iter-47100 loss: 0.2898 validation: 0.970000\n",
            "Iter-47200 loss: 0.1096 validation: 0.970000\n",
            "Iter-47300 loss: 0.2497 validation: 0.970000\n",
            "Iter-47400 loss: 0.2873 validation: 0.970000\n",
            "Iter-47500 loss: 0.2676 validation: 0.970000\n",
            "Iter-47600 loss: 0.1569 validation: 0.970000\n",
            "Iter-47700 loss: 0.1966 validation: 0.970000\n",
            "Iter-47800 loss: 0.2745 validation: 0.970000\n",
            "Iter-47900 loss: 0.1660 validation: 0.970000\n",
            "Iter-48000 loss: 0.2663 validation: 0.970000\n",
            "Iter-48100 loss: 0.1496 validation: 0.970000\n",
            "Iter-48200 loss: 0.1410 validation: 0.970000\n",
            "Iter-48300 loss: 0.3896 validation: 0.970000\n",
            "Iter-48400 loss: 0.1567 validation: 0.970000\n",
            "Iter-48500 loss: 0.2757 validation: 0.970000\n",
            "Iter-48600 loss: 0.1944 validation: 0.970000\n",
            "Iter-48700 loss: 0.2264 validation: 0.970000\n",
            "Iter-48800 loss: 0.1426 validation: 0.980000\n",
            "Iter-48900 loss: 0.1995 validation: 0.970000\n",
            "Iter-49000 loss: 0.1620 validation: 0.980000\n",
            "Iter-49100 loss: 0.1640 validation: 0.970000\n",
            "Iter-49200 loss: 0.1350 validation: 0.970000\n",
            "Iter-49300 loss: 0.1867 validation: 0.970000\n",
            "Iter-49400 loss: 0.1334 validation: 0.970000\n",
            "Iter-49500 loss: 0.1918 validation: 0.970000\n",
            "Iter-49600 loss: 0.2692 validation: 0.980000\n",
            "Iter-49700 loss: 0.1876 validation: 0.970000\n",
            "Iter-49800 loss: 0.1555 validation: 0.970000\n",
            "Iter-49900 loss: 0.1754 validation: 0.970000\n",
            "Iter-50000 loss: 0.3851 validation: 0.970000\n",
            "\n",
            "Best max_norm:  inf best dropout rate for input:  0.85 best dropout rate for hidden layers:  0.5 and result accuracy is:  0.9677\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "o9T44dEU7UxG"
      },
      "source": [
        "Dropout NN + max-norm constraint + ReLU + 3 layers + 1024 hidden units + data augmentation (50,000 weight updates)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "U6y8HKHc7VVV",
        "outputId": "13478ca9-0ee4-4e59-9548-6b3ea4bc04fe"
      },
      "source": [
        "model = 'ff'\n",
        "n_iter = 50000\n",
        "alpha = 1e-3\n",
        "mb_size = 64\n",
        "n_experiment = 1\n",
        "reg = 1e-5\n",
        "print_after = 100\n",
        "p_dropout = [0.8,0.5] #if nlayer=n, len(p_dropout) = n-1\n",
        "loss = 'cross_ent'\n",
        "nonlin = 'relu'\n",
        "solver = 'sgd'\n",
        "max_norm = 4\n",
        "nlayer = 3\n",
        "\n",
        "# Define default candidates\n",
        "candidates = np.array([[3, 3.5, 4],[0.795, 0.8, 0.85],[0.495, 0.5, 0.55]]) # first hyper-param as max-norm, second as drop-out rate for input layer, third as drop-out rate for hidden layers\n",
        "\n",
        "run_mnist(1024, model, n_iter, alpha, mb_size, n_experiment, reg, print_after, p_dropout, loss, nonlin, solver, max_norm, x_train_aug, x_validation_aug, x_test, y_train_aug, y_validation_aug, y_test, candidates, nlayer)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Experimenting on sgd\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  3.0 input drop out rate:  0.795 hidden layer drop out rate:  0.495\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.6900 validation: 0.850000\n",
            "Iter-200 loss: 1.5393 validation: 0.870000\n",
            "Iter-300 loss: 1.4607 validation: 0.890000\n",
            "Iter-400 loss: 1.4291 validation: 0.920000\n",
            "Iter-500 loss: 1.3718 validation: 0.930000\n",
            "\n",
            "Mean accuracy: 0.8929, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  3.0 input drop out rate:  0.795 hidden layer drop out rate:  0.5\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.7746 validation: 0.850000\n",
            "Iter-200 loss: 1.6210 validation: 0.870000\n",
            "Iter-300 loss: 1.4106 validation: 0.900000\n",
            "Iter-400 loss: 1.5196 validation: 0.900000\n",
            "Iter-500 loss: 1.3994 validation: 0.920000\n",
            "\n",
            "Mean accuracy: 0.8903, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  3.0 input drop out rate:  0.795 hidden layer drop out rate:  0.55\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.6522 validation: 0.870000\n",
            "Iter-200 loss: 1.5895 validation: 0.880000\n",
            "Iter-300 loss: 1.4371 validation: 0.900000\n",
            "Iter-400 loss: 1.4422 validation: 0.910000\n",
            "Iter-500 loss: 1.3904 validation: 0.910000\n",
            "\n",
            "Mean accuracy: 0.8986, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  3.0 input drop out rate:  0.8 hidden layer drop out rate:  0.495\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.6628 validation: 0.820000\n",
            "Iter-200 loss: 1.6535 validation: 0.870000\n",
            "Iter-300 loss: 1.4763 validation: 0.890000\n",
            "Iter-400 loss: 1.4039 validation: 0.910000\n",
            "Iter-500 loss: 1.3857 validation: 0.930000\n",
            "\n",
            "Mean accuracy: 0.8924, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  3.0 input drop out rate:  0.8 hidden layer drop out rate:  0.5\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.6589 validation: 0.850000\n",
            "Iter-200 loss: 1.5049 validation: 0.890000\n",
            "Iter-300 loss: 1.4617 validation: 0.910000\n",
            "Iter-400 loss: 1.4302 validation: 0.920000\n",
            "Iter-500 loss: 1.3567 validation: 0.930000\n",
            "\n",
            "Mean accuracy: 0.8956, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  3.0 input drop out rate:  0.8 hidden layer drop out rate:  0.55\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.7262 validation: 0.840000\n",
            "Iter-200 loss: 1.6198 validation: 0.880000\n",
            "Iter-300 loss: 1.5616 validation: 0.900000\n",
            "Iter-400 loss: 1.4759 validation: 0.900000\n",
            "Iter-500 loss: 1.4043 validation: 0.910000\n",
            "\n",
            "Mean accuracy: 0.8931, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  3.0 input drop out rate:  0.85 hidden layer drop out rate:  0.495\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.5649 validation: 0.860000\n",
            "Iter-200 loss: 1.5939 validation: 0.890000\n",
            "Iter-300 loss: 1.3352 validation: 0.890000\n",
            "Iter-400 loss: 1.3781 validation: 0.920000\n",
            "Iter-500 loss: 1.2927 validation: 0.920000\n",
            "\n",
            "Mean accuracy: 0.8994, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  3.0 input drop out rate:  0.85 hidden layer drop out rate:  0.5\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.6008 validation: 0.860000\n",
            "Iter-200 loss: 1.4688 validation: 0.890000\n",
            "Iter-300 loss: 1.4548 validation: 0.920000\n",
            "Iter-400 loss: 1.3909 validation: 0.920000\n",
            "Iter-500 loss: 1.3296 validation: 0.930000\n",
            "\n",
            "Mean accuracy: 0.8974, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  3.0 input drop out rate:  0.85 hidden layer drop out rate:  0.55\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.6217 validation: 0.850000\n",
            "Iter-200 loss: 1.5205 validation: 0.880000\n",
            "Iter-300 loss: 1.5233 validation: 0.910000\n",
            "Iter-400 loss: 1.4803 validation: 0.920000\n",
            "Iter-500 loss: 1.3373 validation: 0.930000\n",
            "\n",
            "Mean accuracy: 0.8968, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  3.5 input drop out rate:  0.795 hidden layer drop out rate:  0.495\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.4837 validation: 0.850000\n",
            "Iter-200 loss: 1.4758 validation: 0.890000\n",
            "Iter-300 loss: 1.2129 validation: 0.900000\n",
            "Iter-400 loss: 1.2101 validation: 0.910000\n",
            "Iter-500 loss: 1.1331 validation: 0.930000\n",
            "\n",
            "Mean accuracy: 0.9003, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  3.5 input drop out rate:  0.795 hidden layer drop out rate:  0.5\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.5374 validation: 0.850000\n",
            "Iter-200 loss: 1.3704 validation: 0.880000\n",
            "Iter-300 loss: 1.2271 validation: 0.900000\n",
            "Iter-400 loss: 1.1796 validation: 0.920000\n",
            "Iter-500 loss: 1.2283 validation: 0.930000\n",
            "\n",
            "Mean accuracy: 0.8945, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  3.5 input drop out rate:  0.795 hidden layer drop out rate:  0.55\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.4987 validation: 0.860000\n",
            "Iter-200 loss: 1.3890 validation: 0.900000\n",
            "Iter-300 loss: 1.1414 validation: 0.910000\n",
            "Iter-400 loss: 1.1911 validation: 0.930000\n",
            "Iter-500 loss: 1.1951 validation: 0.930000\n",
            "\n",
            "Mean accuracy: 0.8966, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  3.5 input drop out rate:  0.8 hidden layer drop out rate:  0.495\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.6036 validation: 0.840000\n",
            "Iter-200 loss: 1.2916 validation: 0.900000\n",
            "Iter-300 loss: 1.1505 validation: 0.920000\n",
            "Iter-400 loss: 1.1485 validation: 0.920000\n",
            "Iter-500 loss: 1.1699 validation: 0.930000\n",
            "\n",
            "Mean accuracy: 0.8969, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  3.5 input drop out rate:  0.8 hidden layer drop out rate:  0.5\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.5926 validation: 0.870000\n",
            "Iter-200 loss: 1.2908 validation: 0.900000\n",
            "Iter-300 loss: 1.2704 validation: 0.900000\n",
            "Iter-400 loss: 1.1478 validation: 0.910000\n",
            "Iter-500 loss: 1.1153 validation: 0.920000\n",
            "\n",
            "Mean accuracy: 0.9006, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  3.5 input drop out rate:  0.8 hidden layer drop out rate:  0.55\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.5105 validation: 0.860000\n",
            "Iter-200 loss: 1.3188 validation: 0.900000\n",
            "Iter-300 loss: 1.2084 validation: 0.910000\n",
            "Iter-400 loss: 1.2487 validation: 0.920000\n",
            "Iter-500 loss: 1.0665 validation: 0.940000\n",
            "\n",
            "Mean accuracy: 0.9022, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  3.5 input drop out rate:  0.85 hidden layer drop out rate:  0.495\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.4686 validation: 0.890000\n",
            "Iter-200 loss: 1.4515 validation: 0.900000\n",
            "Iter-300 loss: 1.2400 validation: 0.920000\n",
            "Iter-400 loss: 1.1309 validation: 0.930000\n",
            "Iter-500 loss: 1.1028 validation: 0.940000\n",
            "\n",
            "Mean accuracy: 0.9012, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  3.5 input drop out rate:  0.85 hidden layer drop out rate:  0.5\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.4096 validation: 0.860000\n",
            "Iter-200 loss: 1.2456 validation: 0.900000\n",
            "Iter-300 loss: 1.2447 validation: 0.930000\n",
            "Iter-400 loss: 1.2188 validation: 0.920000\n",
            "Iter-500 loss: 1.0737 validation: 0.930000\n",
            "\n",
            "Mean accuracy: 0.9032, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  3.5 input drop out rate:  0.85 hidden layer drop out rate:  0.55\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.5742 validation: 0.850000\n",
            "Iter-200 loss: 1.2821 validation: 0.880000\n",
            "Iter-300 loss: 1.2990 validation: 0.910000\n",
            "Iter-400 loss: 1.0793 validation: 0.950000\n",
            "Iter-500 loss: 1.0827 validation: 0.950000\n",
            "\n",
            "Mean accuracy: 0.9042, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  4.0 input drop out rate:  0.795 hidden layer drop out rate:  0.495\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.4264 validation: 0.860000\n",
            "Iter-200 loss: 1.1839 validation: 0.890000\n",
            "Iter-300 loss: 1.1308 validation: 0.910000\n",
            "Iter-400 loss: 1.0512 validation: 0.930000\n",
            "Iter-500 loss: 1.0135 validation: 0.950000\n",
            "\n",
            "Mean accuracy: 0.9045, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  4.0 input drop out rate:  0.795 hidden layer drop out rate:  0.5\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.3707 validation: 0.850000\n",
            "Iter-200 loss: 1.1570 validation: 0.900000\n",
            "Iter-300 loss: 1.0779 validation: 0.910000\n",
            "Iter-400 loss: 0.9423 validation: 0.920000\n",
            "Iter-500 loss: 0.9261 validation: 0.940000\n",
            "\n",
            "Mean accuracy: 0.9034, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  4.0 input drop out rate:  0.795 hidden layer drop out rate:  0.55\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.2576 validation: 0.850000\n",
            "Iter-200 loss: 1.0888 validation: 0.890000\n",
            "Iter-300 loss: 0.9975 validation: 0.910000\n",
            "Iter-400 loss: 1.0846 validation: 0.920000\n",
            "Iter-500 loss: 0.8850 validation: 0.950000\n",
            "\n",
            "Mean accuracy: 0.9054, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  4.0 input drop out rate:  0.8 hidden layer drop out rate:  0.495\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.2781 validation: 0.840000\n",
            "Iter-200 loss: 1.0294 validation: 0.890000\n",
            "Iter-300 loss: 1.0346 validation: 0.920000\n",
            "Iter-400 loss: 1.0507 validation: 0.920000\n",
            "Iter-500 loss: 1.0381 validation: 0.950000\n",
            "\n",
            "Mean accuracy: 0.9047, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  4.0 input drop out rate:  0.8 hidden layer drop out rate:  0.5\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.3545 validation: 0.860000\n",
            "Iter-200 loss: 1.1319 validation: 0.900000\n",
            "Iter-300 loss: 1.0040 validation: 0.900000\n",
            "Iter-400 loss: 0.9606 validation: 0.920000\n",
            "Iter-500 loss: 0.9924 validation: 0.940000\n",
            "\n",
            "Mean accuracy: 0.9042, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  4.0 input drop out rate:  0.8 hidden layer drop out rate:  0.55\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.2577 validation: 0.850000\n",
            "Iter-200 loss: 1.2288 validation: 0.880000\n",
            "Iter-300 loss: 1.1272 validation: 0.900000\n",
            "Iter-400 loss: 0.8577 validation: 0.920000\n",
            "Iter-500 loss: 0.9169 validation: 0.940000\n",
            "\n",
            "Mean accuracy: 0.9009, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  4.0 input drop out rate:  0.85 hidden layer drop out rate:  0.495\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.3470 validation: 0.850000\n",
            "Iter-200 loss: 1.1454 validation: 0.890000\n",
            "Iter-300 loss: 0.9393 validation: 0.920000\n",
            "Iter-400 loss: 0.9303 validation: 0.930000\n",
            "Iter-500 loss: 0.8391 validation: 0.950000\n",
            "\n",
            "Mean accuracy: 0.9074, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  4.0 input drop out rate:  0.85 hidden layer drop out rate:  0.5\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.2329 validation: 0.860000\n",
            "Iter-200 loss: 1.1349 validation: 0.890000\n",
            "Iter-300 loss: 1.0636 validation: 0.910000\n",
            "Iter-400 loss: 0.9885 validation: 0.940000\n",
            "Iter-500 loss: 0.9280 validation: 0.930000\n",
            "\n",
            "Mean accuracy: 0.9056, std: 0.0000\n",
            "\n",
            "Experiment on hyper-parm candidates. max_norm:  4.0 input drop out rate:  0.85 hidden layer drop out rate:  0.55\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.2743 validation: 0.860000\n",
            "Iter-200 loss: 1.0328 validation: 0.900000\n",
            "Iter-300 loss: 1.0356 validation: 0.910000\n",
            "Iter-400 loss: 0.9169 validation: 0.940000\n",
            "Iter-500 loss: 1.0792 validation: 0.940000\n",
            "\n",
            "Mean accuracy: 0.9065, std: 0.0000\n",
            "\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.6100 validation: 0.860000\n",
            "Iter-200 loss: 1.4049 validation: 0.870000\n",
            "Iter-300 loss: 1.3049 validation: 0.900000\n",
            "Iter-400 loss: 1.1689 validation: 0.900000\n",
            "Iter-500 loss: 1.0422 validation: 0.900000\n",
            "Iter-600 loss: 1.0741 validation: 0.920000\n",
            "Iter-700 loss: 0.8786 validation: 0.920000\n",
            "Iter-800 loss: 0.9465 validation: 0.930000\n",
            "Iter-900 loss: 0.9399 validation: 0.930000\n",
            "Iter-1000 loss: 0.8833 validation: 0.930000\n",
            "Iter-1100 loss: 0.8692 validation: 0.930000\n",
            "Iter-1200 loss: 0.8371 validation: 0.940000\n",
            "Iter-1300 loss: 0.8092 validation: 0.940000\n",
            "Iter-1400 loss: 0.8103 validation: 0.950000\n",
            "Iter-1500 loss: 0.9021 validation: 0.950000\n",
            "Iter-1600 loss: 0.8128 validation: 0.940000\n",
            "Iter-1700 loss: 0.7379 validation: 0.940000\n",
            "Iter-1800 loss: 0.9027 validation: 0.950000\n",
            "Iter-1900 loss: 0.6773 validation: 0.950000\n",
            "Iter-2000 loss: 0.6809 validation: 0.950000\n",
            "Iter-2100 loss: 0.7441 validation: 0.950000\n",
            "Iter-2200 loss: 0.6189 validation: 0.950000\n",
            "Iter-2300 loss: 0.6824 validation: 0.950000\n",
            "Iter-2400 loss: 0.7150 validation: 0.950000\n",
            "Iter-2500 loss: 0.6323 validation: 0.950000\n",
            "Iter-2600 loss: 0.7099 validation: 0.950000\n",
            "Iter-2700 loss: 0.7065 validation: 0.950000\n",
            "Iter-2800 loss: 0.7053 validation: 0.950000\n",
            "Iter-2900 loss: 0.5821 validation: 0.950000\n",
            "Iter-3000 loss: 0.6538 validation: 0.950000\n",
            "Iter-3100 loss: 0.7683 validation: 0.950000\n",
            "Iter-3200 loss: 0.6821 validation: 0.950000\n",
            "Iter-3300 loss: 0.6636 validation: 0.950000\n",
            "Iter-3400 loss: 0.6232 validation: 0.950000\n",
            "Iter-3500 loss: 0.6582 validation: 0.950000\n",
            "Iter-3600 loss: 0.6235 validation: 0.950000\n",
            "Iter-3700 loss: 0.6613 validation: 0.950000\n",
            "Iter-3800 loss: 0.6414 validation: 0.950000\n",
            "Iter-3900 loss: 0.5592 validation: 0.950000\n",
            "Iter-4000 loss: 0.6613 validation: 0.960000\n",
            "Iter-4100 loss: 0.4459 validation: 0.950000\n",
            "Iter-4200 loss: 0.6576 validation: 0.960000\n",
            "Iter-4300 loss: 0.6530 validation: 0.970000\n",
            "Iter-4400 loss: 0.7180 validation: 0.960000\n",
            "Iter-4500 loss: 0.4856 validation: 0.950000\n",
            "Iter-4600 loss: 0.5732 validation: 0.970000\n",
            "Iter-4700 loss: 0.4913 validation: 0.950000\n",
            "Iter-4800 loss: 0.5989 validation: 0.960000\n",
            "Iter-4900 loss: 0.5671 validation: 0.970000\n",
            "Iter-5000 loss: 0.5451 validation: 0.970000\n",
            "Iter-5100 loss: 0.4485 validation: 0.960000\n",
            "Iter-5200 loss: 0.5552 validation: 0.950000\n",
            "Iter-5300 loss: 0.5389 validation: 0.960000\n",
            "Iter-5400 loss: 0.4989 validation: 0.970000\n",
            "Iter-5500 loss: 0.4717 validation: 0.970000\n",
            "Iter-5600 loss: 0.4595 validation: 0.970000\n",
            "Iter-5700 loss: 0.5349 validation: 0.970000\n",
            "Iter-5800 loss: 0.4900 validation: 0.970000\n",
            "Iter-5900 loss: 0.5081 validation: 0.970000\n",
            "Iter-6000 loss: 0.6005 validation: 0.970000\n",
            "Iter-6100 loss: 0.5073 validation: 0.970000\n",
            "Iter-6200 loss: 0.5713 validation: 0.970000\n",
            "Iter-6300 loss: 0.5223 validation: 0.970000\n",
            "Iter-6400 loss: 0.5119 validation: 0.970000\n",
            "Iter-6500 loss: 0.5744 validation: 0.970000\n",
            "Iter-6600 loss: 0.4471 validation: 0.970000\n",
            "Iter-6700 loss: 0.5094 validation: 0.970000\n",
            "Iter-6800 loss: 0.5376 validation: 0.960000\n",
            "Iter-6900 loss: 0.5436 validation: 0.970000\n",
            "Iter-7000 loss: 0.4805 validation: 0.970000\n",
            "Iter-7100 loss: 0.4760 validation: 0.970000\n",
            "Iter-7200 loss: 0.4298 validation: 0.970000\n",
            "Iter-7300 loss: 0.5090 validation: 0.970000\n",
            "Iter-7400 loss: 0.4283 validation: 0.970000\n",
            "Iter-7500 loss: 0.5507 validation: 0.970000\n",
            "Iter-7600 loss: 0.4743 validation: 0.970000\n",
            "Iter-7700 loss: 0.4326 validation: 0.970000\n",
            "Iter-7800 loss: 0.4202 validation: 0.970000\n",
            "Iter-7900 loss: 0.4190 validation: 0.970000\n",
            "Iter-8000 loss: 0.4542 validation: 0.970000\n",
            "Iter-8100 loss: 0.4041 validation: 0.970000\n",
            "Iter-8200 loss: 0.4345 validation: 0.970000\n",
            "Iter-8300 loss: 0.6020 validation: 0.970000\n",
            "Iter-8400 loss: 0.4089 validation: 0.970000\n",
            "Iter-8500 loss: 0.4608 validation: 0.970000\n",
            "Iter-8600 loss: 0.4231 validation: 0.970000\n",
            "Iter-8700 loss: 0.4327 validation: 0.970000\n",
            "Iter-8800 loss: 0.5275 validation: 0.970000\n",
            "Iter-8900 loss: 0.4990 validation: 0.970000\n",
            "Iter-9000 loss: 0.4998 validation: 0.970000\n",
            "Iter-9100 loss: 0.4326 validation: 0.970000\n",
            "Iter-9200 loss: 0.4986 validation: 0.970000\n",
            "Iter-9300 loss: 0.4779 validation: 0.970000\n",
            "Iter-9400 loss: 0.4339 validation: 0.970000\n",
            "Iter-9500 loss: 0.3818 validation: 0.970000\n",
            "Iter-9600 loss: 0.4335 validation: 0.970000\n",
            "Iter-9700 loss: 0.3914 validation: 0.970000\n",
            "Iter-9800 loss: 0.4371 validation: 0.970000\n",
            "Iter-9900 loss: 0.3626 validation: 0.970000\n",
            "Iter-10000 loss: 0.4725 validation: 0.970000\n",
            "Iter-10100 loss: 0.3809 validation: 0.970000\n",
            "Iter-10200 loss: 0.3571 validation: 0.970000\n",
            "Iter-10300 loss: 0.3765 validation: 0.970000\n",
            "Iter-10400 loss: 0.3427 validation: 0.970000\n",
            "Iter-10500 loss: 0.4058 validation: 0.970000\n",
            "Iter-10600 loss: 0.4518 validation: 0.970000\n",
            "Iter-10700 loss: 0.3029 validation: 0.970000\n",
            "Iter-10800 loss: 0.3737 validation: 0.970000\n",
            "Iter-10900 loss: 0.3730 validation: 0.970000\n",
            "Iter-11000 loss: 0.3425 validation: 0.970000\n",
            "Iter-11100 loss: 0.4016 validation: 0.970000\n",
            "Iter-11200 loss: 0.3979 validation: 0.970000\n",
            "Iter-11300 loss: 0.3699 validation: 0.970000\n",
            "Iter-11400 loss: 0.3431 validation: 0.970000\n",
            "Iter-11500 loss: 0.3663 validation: 0.970000\n",
            "Iter-11600 loss: 0.3623 validation: 0.970000\n",
            "Iter-11700 loss: 0.4978 validation: 0.970000\n",
            "Iter-11800 loss: 0.3224 validation: 0.970000\n",
            "Iter-11900 loss: 0.4291 validation: 0.970000\n",
            "Iter-12000 loss: 0.4841 validation: 0.970000\n",
            "Iter-12100 loss: 0.3691 validation: 0.970000\n",
            "Iter-12200 loss: 0.3556 validation: 0.970000\n",
            "Iter-12300 loss: 0.3040 validation: 0.970000\n",
            "Iter-12400 loss: 0.2647 validation: 0.970000\n",
            "Iter-12500 loss: 0.3354 validation: 0.970000\n",
            "Iter-12600 loss: 0.3557 validation: 0.970000\n",
            "Iter-12700 loss: 0.3912 validation: 0.970000\n",
            "Iter-12800 loss: 0.4005 validation: 0.970000\n",
            "Iter-12900 loss: 0.3766 validation: 0.970000\n",
            "Iter-13000 loss: 0.3528 validation: 0.970000\n",
            "Iter-13100 loss: 0.4245 validation: 0.970000\n",
            "Iter-13200 loss: 0.3463 validation: 0.970000\n",
            "Iter-13300 loss: 0.4142 validation: 0.970000\n",
            "Iter-13400 loss: 0.3269 validation: 0.970000\n",
            "Iter-13500 loss: 0.3293 validation: 0.970000\n",
            "Iter-13600 loss: 0.2944 validation: 0.970000\n",
            "Iter-13700 loss: 0.2873 validation: 0.970000\n",
            "Iter-13800 loss: 0.3236 validation: 0.970000\n",
            "Iter-13900 loss: 0.2560 validation: 0.970000\n",
            "Iter-14000 loss: 0.3602 validation: 0.970000\n",
            "Iter-14100 loss: 0.2900 validation: 0.970000\n",
            "Iter-14200 loss: 0.2600 validation: 0.970000\n",
            "Iter-14300 loss: 0.3614 validation: 0.970000\n",
            "Iter-14400 loss: 0.3540 validation: 0.970000\n",
            "Iter-14500 loss: 0.2760 validation: 0.970000\n",
            "Iter-14600 loss: 0.3121 validation: 0.970000\n",
            "Iter-14700 loss: 0.4122 validation: 0.970000\n",
            "Iter-14800 loss: 0.4639 validation: 0.970000\n",
            "Iter-14900 loss: 0.3300 validation: 0.970000\n",
            "Iter-15000 loss: 0.2996 validation: 0.970000\n",
            "Iter-15100 loss: 0.4167 validation: 0.970000\n",
            "Iter-15200 loss: 0.4009 validation: 0.970000\n",
            "Iter-15300 loss: 0.3119 validation: 0.970000\n",
            "Iter-15400 loss: 0.2960 validation: 0.970000\n",
            "Iter-15500 loss: 0.4530 validation: 0.970000\n",
            "Iter-15600 loss: 0.3004 validation: 0.970000\n",
            "Iter-15700 loss: 0.2637 validation: 0.970000\n",
            "Iter-15800 loss: 0.3742 validation: 0.970000\n",
            "Iter-15900 loss: 0.2677 validation: 0.970000\n",
            "Iter-16000 loss: 0.3098 validation: 0.970000\n",
            "Iter-16100 loss: 0.3170 validation: 0.970000\n",
            "Iter-16200 loss: 0.3615 validation: 0.970000\n",
            "Iter-16300 loss: 0.3369 validation: 0.970000\n",
            "Iter-16400 loss: 0.3135 validation: 0.970000\n",
            "Iter-16500 loss: 0.3648 validation: 0.970000\n",
            "Iter-16600 loss: 0.4006 validation: 0.970000\n",
            "Iter-16700 loss: 0.3078 validation: 0.970000\n",
            "Iter-16800 loss: 0.3629 validation: 0.970000\n",
            "Iter-16900 loss: 0.2887 validation: 0.970000\n",
            "Iter-17000 loss: 0.5059 validation: 0.970000\n",
            "Iter-17100 loss: 0.3177 validation: 0.970000\n",
            "Iter-17200 loss: 0.3220 validation: 0.970000\n",
            "Iter-17300 loss: 0.3185 validation: 0.970000\n",
            "Iter-17400 loss: 0.3684 validation: 0.970000\n",
            "Iter-17500 loss: 0.2129 validation: 0.970000\n",
            "Iter-17600 loss: 0.2656 validation: 0.970000\n",
            "Iter-17700 loss: 0.2952 validation: 0.970000\n",
            "Iter-17800 loss: 0.2557 validation: 0.970000\n",
            "Iter-17900 loss: 0.3246 validation: 0.970000\n",
            "Iter-18000 loss: 0.2066 validation: 0.970000\n",
            "Iter-18100 loss: 0.3190 validation: 0.970000\n",
            "Iter-18200 loss: 0.2642 validation: 0.970000\n",
            "Iter-18300 loss: 0.3212 validation: 0.970000\n",
            "Iter-18400 loss: 0.2717 validation: 0.970000\n",
            "Iter-18500 loss: 0.3178 validation: 0.970000\n",
            "Iter-18600 loss: 0.2892 validation: 0.970000\n",
            "Iter-18700 loss: 0.3056 validation: 0.970000\n",
            "Iter-18800 loss: 0.2989 validation: 0.970000\n",
            "Iter-18900 loss: 0.3242 validation: 0.970000\n",
            "Iter-19000 loss: 0.3590 validation: 0.970000\n",
            "Iter-19100 loss: 0.2382 validation: 0.970000\n",
            "Iter-19200 loss: 0.2723 validation: 0.970000\n",
            "Iter-19300 loss: 0.2685 validation: 0.970000\n",
            "Iter-19400 loss: 0.2773 validation: 0.970000\n",
            "Iter-19500 loss: 0.2854 validation: 0.970000\n",
            "Iter-19600 loss: 0.2755 validation: 0.970000\n",
            "Iter-19700 loss: 0.2538 validation: 0.970000\n",
            "Iter-19800 loss: 0.3216 validation: 0.970000\n",
            "Iter-19900 loss: 0.3391 validation: 0.970000\n",
            "Iter-20000 loss: 0.2312 validation: 0.970000\n",
            "Iter-20100 loss: 0.3017 validation: 0.970000\n",
            "Iter-20200 loss: 0.1872 validation: 0.970000\n",
            "Iter-20300 loss: 0.3050 validation: 0.970000\n",
            "Iter-20400 loss: 0.2000 validation: 0.970000\n",
            "Iter-20500 loss: 0.2880 validation: 0.970000\n",
            "Iter-20600 loss: 0.3171 validation: 0.970000\n",
            "Iter-20700 loss: 0.2625 validation: 0.970000\n",
            "Iter-20800 loss: 0.3305 validation: 0.970000\n",
            "Iter-20900 loss: 0.2197 validation: 0.970000\n",
            "Iter-21000 loss: 0.2455 validation: 0.970000\n",
            "Iter-21100 loss: 0.2729 validation: 0.970000\n",
            "Iter-21200 loss: 0.3143 validation: 0.970000\n",
            "Iter-21300 loss: 0.3031 validation: 0.970000\n",
            "Iter-21400 loss: 0.4075 validation: 0.970000\n",
            "Iter-21500 loss: 0.2633 validation: 0.970000\n",
            "Iter-21600 loss: 0.2066 validation: 0.970000\n",
            "Iter-21700 loss: 0.2918 validation: 0.970000\n",
            "Iter-21800 loss: 0.3245 validation: 0.970000\n",
            "Iter-21900 loss: 0.3507 validation: 0.970000\n",
            "Iter-22000 loss: 0.2202 validation: 0.970000\n",
            "Iter-22100 loss: 0.2711 validation: 0.970000\n",
            "Iter-22200 loss: 0.2494 validation: 0.970000\n",
            "Iter-22300 loss: 0.3095 validation: 0.970000\n",
            "Iter-22400 loss: 0.2327 validation: 0.970000\n",
            "Iter-22500 loss: 0.2177 validation: 0.970000\n",
            "Iter-22600 loss: 0.2392 validation: 0.970000\n",
            "Iter-22700 loss: 0.2610 validation: 0.970000\n",
            "Iter-22800 loss: 0.2920 validation: 0.970000\n",
            "Iter-22900 loss: 0.2204 validation: 0.970000\n",
            "Iter-23000 loss: 0.3668 validation: 0.970000\n",
            "Iter-23100 loss: 0.2812 validation: 0.970000\n",
            "Iter-23200 loss: 0.3200 validation: 0.970000\n",
            "Iter-23300 loss: 0.2349 validation: 0.970000\n",
            "Iter-23400 loss: 0.2683 validation: 0.970000\n",
            "Iter-23500 loss: 0.2617 validation: 0.970000\n",
            "Iter-23600 loss: 0.1624 validation: 0.970000\n",
            "Iter-23700 loss: 0.2094 validation: 0.970000\n",
            "Iter-23800 loss: 0.1898 validation: 0.970000\n",
            "Iter-23900 loss: 0.2745 validation: 0.970000\n",
            "Iter-24000 loss: 0.3248 validation: 0.970000\n",
            "Iter-24100 loss: 0.2331 validation: 0.970000\n",
            "Iter-24200 loss: 0.2342 validation: 0.970000\n",
            "Iter-24300 loss: 0.3176 validation: 0.970000\n",
            "Iter-24400 loss: 0.2350 validation: 0.970000\n",
            "Iter-24500 loss: 0.2156 validation: 0.970000\n",
            "Iter-24600 loss: 0.2882 validation: 0.970000\n",
            "Iter-24700 loss: 0.2287 validation: 0.970000\n",
            "Iter-24800 loss: 0.2116 validation: 0.970000\n",
            "Iter-24900 loss: 0.3189 validation: 0.970000\n",
            "Iter-25000 loss: 0.2366 validation: 0.970000\n",
            "Iter-25100 loss: 0.2821 validation: 0.970000\n",
            "Iter-25200 loss: 0.3300 validation: 0.970000\n",
            "Iter-25300 loss: 0.2687 validation: 0.970000\n",
            "Iter-25400 loss: 0.3187 validation: 0.970000\n",
            "Iter-25500 loss: 0.2925 validation: 0.970000\n",
            "Iter-25600 loss: 0.2882 validation: 0.970000\n",
            "Iter-25700 loss: 0.2127 validation: 0.970000\n",
            "Iter-25800 loss: 0.2839 validation: 0.970000\n",
            "Iter-25900 loss: 0.3026 validation: 0.970000\n",
            "Iter-26000 loss: 0.1932 validation: 0.970000\n",
            "Iter-26100 loss: 0.2361 validation: 0.970000\n",
            "Iter-26200 loss: 0.1909 validation: 0.970000\n",
            "Iter-26300 loss: 0.2735 validation: 0.970000\n",
            "Iter-26400 loss: 0.2084 validation: 0.970000\n",
            "Iter-26500 loss: 0.2270 validation: 0.970000\n",
            "Iter-26600 loss: 0.3364 validation: 0.970000\n",
            "Iter-26700 loss: 0.2273 validation: 0.970000\n",
            "Iter-26800 loss: 0.2606 validation: 0.970000\n",
            "Iter-26900 loss: 0.2154 validation: 0.970000\n",
            "Iter-27000 loss: 0.2216 validation: 0.970000\n",
            "Iter-27100 loss: 0.2414 validation: 0.970000\n",
            "Iter-27200 loss: 0.2530 validation: 0.970000\n",
            "Iter-27300 loss: 0.2335 validation: 0.970000\n",
            "Iter-27400 loss: 0.2033 validation: 0.970000\n",
            "Iter-27500 loss: 0.2507 validation: 0.970000\n",
            "Iter-27600 loss: 0.2126 validation: 0.970000\n",
            "Iter-27700 loss: 0.2968 validation: 0.970000\n",
            "Iter-27800 loss: 0.2525 validation: 0.970000\n",
            "Iter-27900 loss: 0.2877 validation: 0.970000\n",
            "Iter-28000 loss: 0.1897 validation: 0.970000\n",
            "Iter-28100 loss: 0.2157 validation: 0.970000\n",
            "Iter-28200 loss: 0.2916 validation: 0.970000\n",
            "Iter-28300 loss: 0.2201 validation: 0.970000\n",
            "Iter-28400 loss: 0.3045 validation: 0.970000\n",
            "Iter-28500 loss: 0.2683 validation: 0.970000\n",
            "Iter-28600 loss: 0.1975 validation: 0.970000\n",
            "Iter-28700 loss: 0.2437 validation: 0.970000\n",
            "Iter-28800 loss: 0.2019 validation: 0.970000\n",
            "Iter-28900 loss: 0.2203 validation: 0.970000\n",
            "Iter-29000 loss: 0.2610 validation: 0.970000\n",
            "Iter-29100 loss: 0.2212 validation: 0.970000\n",
            "Iter-29200 loss: 0.2300 validation: 0.970000\n",
            "Iter-29300 loss: 0.2403 validation: 0.970000\n",
            "Iter-29400 loss: 0.1813 validation: 0.970000\n",
            "Iter-29500 loss: 0.2442 validation: 0.970000\n",
            "Iter-29600 loss: 0.3367 validation: 0.970000\n",
            "Iter-29700 loss: 0.2698 validation: 0.970000\n",
            "Iter-29800 loss: 0.2077 validation: 0.970000\n",
            "Iter-29900 loss: 0.1956 validation: 0.970000\n",
            "Iter-30000 loss: 0.2141 validation: 0.970000\n",
            "Iter-30100 loss: 0.2598 validation: 0.970000\n",
            "Iter-30200 loss: 0.2480 validation: 0.970000\n",
            "Iter-30300 loss: 0.2753 validation: 0.970000\n",
            "Iter-30400 loss: 0.2015 validation: 0.970000\n",
            "Iter-30500 loss: 0.1972 validation: 0.970000\n",
            "Iter-30600 loss: 0.2163 validation: 0.970000\n",
            "Iter-30700 loss: 0.2536 validation: 0.970000\n",
            "Iter-30800 loss: 0.3208 validation: 0.970000\n",
            "Iter-30900 loss: 0.1950 validation: 0.970000\n",
            "Iter-31000 loss: 0.2419 validation: 0.970000\n",
            "Iter-31100 loss: 0.1819 validation: 0.970000\n",
            "Iter-31200 loss: 0.2224 validation: 0.970000\n",
            "Iter-31300 loss: 0.2348 validation: 0.970000\n",
            "Iter-31400 loss: 0.1873 validation: 0.970000\n",
            "Iter-31500 loss: 0.2193 validation: 0.970000\n",
            "Iter-31600 loss: 0.2070 validation: 0.970000\n",
            "Iter-31700 loss: 0.1722 validation: 0.970000\n",
            "Iter-31800 loss: 0.2161 validation: 0.970000\n",
            "Iter-31900 loss: 0.2179 validation: 0.970000\n",
            "Iter-32000 loss: 0.3123 validation: 0.970000\n",
            "Iter-32100 loss: 0.2362 validation: 0.970000\n",
            "Iter-32200 loss: 0.2095 validation: 0.970000\n",
            "Iter-32300 loss: 0.1772 validation: 0.970000\n",
            "Iter-32400 loss: 0.1981 validation: 0.970000\n",
            "Iter-32500 loss: 0.1749 validation: 0.970000\n",
            "Iter-32600 loss: 0.1616 validation: 0.970000\n",
            "Iter-32700 loss: 0.2087 validation: 0.970000\n",
            "Iter-32800 loss: 0.2407 validation: 0.970000\n",
            "Iter-32900 loss: 0.2574 validation: 0.970000\n",
            "Iter-33000 loss: 0.1871 validation: 0.970000\n",
            "Iter-33100 loss: 0.2312 validation: 0.970000\n",
            "Iter-33200 loss: 0.2708 validation: 0.970000\n",
            "Iter-33300 loss: 0.2015 validation: 0.970000\n",
            "Iter-33400 loss: 0.2325 validation: 0.970000\n",
            "Iter-33500 loss: 0.2073 validation: 0.970000\n",
            "Iter-33600 loss: 0.2428 validation: 0.970000\n",
            "Iter-33700 loss: 0.3226 validation: 0.970000\n",
            "Iter-33800 loss: 0.1896 validation: 0.970000\n",
            "Iter-33900 loss: 0.2056 validation: 0.970000\n",
            "Iter-34000 loss: 0.2368 validation: 0.970000\n",
            "Iter-34100 loss: 0.2090 validation: 0.970000\n",
            "Iter-34200 loss: 0.1995 validation: 0.980000\n",
            "Iter-34300 loss: 0.2161 validation: 0.970000\n",
            "Iter-34400 loss: 0.2386 validation: 0.970000\n",
            "Iter-34500 loss: 0.2353 validation: 0.970000\n",
            "Iter-34600 loss: 0.2372 validation: 0.970000\n",
            "Iter-34700 loss: 0.1884 validation: 0.980000\n",
            "Iter-34800 loss: 0.1968 validation: 0.970000\n",
            "Iter-34900 loss: 0.2111 validation: 0.970000\n",
            "Iter-35000 loss: 0.2428 validation: 0.980000\n",
            "Iter-35100 loss: 0.2256 validation: 0.980000\n",
            "Iter-35200 loss: 0.2311 validation: 0.970000\n",
            "Iter-35300 loss: 0.2676 validation: 0.980000\n",
            "Iter-35400 loss: 0.2556 validation: 0.980000\n",
            "Iter-35500 loss: 0.2392 validation: 0.980000\n",
            "Iter-35600 loss: 0.1838 validation: 0.980000\n",
            "Iter-35700 loss: 0.2027 validation: 0.970000\n",
            "Iter-35800 loss: 0.2628 validation: 0.980000\n",
            "Iter-35900 loss: 0.2064 validation: 0.970000\n",
            "Iter-36000 loss: 0.2487 validation: 0.980000\n",
            "Iter-36100 loss: 0.2712 validation: 0.980000\n",
            "Iter-36200 loss: 0.2638 validation: 0.980000\n",
            "Iter-36300 loss: 0.1983 validation: 0.970000\n",
            "Iter-36400 loss: 0.2013 validation: 0.980000\n",
            "Iter-36500 loss: 0.1814 validation: 0.980000\n",
            "Iter-36600 loss: 0.2528 validation: 0.980000\n",
            "Iter-36700 loss: 0.1764 validation: 0.980000\n",
            "Iter-36800 loss: 0.2208 validation: 0.970000\n",
            "Iter-36900 loss: 0.1687 validation: 0.970000\n",
            "Iter-37000 loss: 0.2318 validation: 0.970000\n",
            "Iter-37100 loss: 0.1484 validation: 0.980000\n",
            "Iter-37200 loss: 0.2028 validation: 0.980000\n",
            "Iter-37300 loss: 0.1974 validation: 0.980000\n",
            "Iter-37400 loss: 0.2361 validation: 0.980000\n",
            "Iter-37500 loss: 0.1846 validation: 0.970000\n",
            "Iter-37600 loss: 0.1803 validation: 0.970000\n",
            "Iter-37700 loss: 0.2411 validation: 0.970000\n",
            "Iter-37800 loss: 0.2248 validation: 0.970000\n",
            "Iter-37900 loss: 0.2318 validation: 0.980000\n",
            "Iter-38000 loss: 0.2777 validation: 0.980000\n",
            "Iter-38100 loss: 0.2525 validation: 0.980000\n",
            "Iter-38200 loss: 0.2408 validation: 0.980000\n",
            "Iter-38300 loss: 0.1562 validation: 0.980000\n",
            "Iter-38400 loss: 0.2486 validation: 0.980000\n",
            "Iter-38500 loss: 0.1921 validation: 0.980000\n",
            "Iter-38600 loss: 0.1901 validation: 0.980000\n",
            "Iter-38700 loss: 0.1611 validation: 0.980000\n",
            "Iter-38800 loss: 0.2312 validation: 0.980000\n",
            "Iter-38900 loss: 0.1704 validation: 0.990000\n",
            "Iter-39000 loss: 0.1753 validation: 0.980000\n",
            "Iter-39100 loss: 0.1699 validation: 0.970000\n",
            "Iter-39200 loss: 0.2341 validation: 0.980000\n",
            "Iter-39300 loss: 0.1834 validation: 0.980000\n",
            "Iter-39400 loss: 0.2155 validation: 0.980000\n",
            "Iter-39500 loss: 0.2190 validation: 0.980000\n",
            "Iter-39600 loss: 0.1535 validation: 0.980000\n",
            "Iter-39700 loss: 0.2056 validation: 0.980000\n",
            "Iter-39800 loss: 0.1672 validation: 0.980000\n",
            "Iter-39900 loss: 0.2686 validation: 0.980000\n",
            "Iter-40000 loss: 0.1845 validation: 0.980000\n",
            "Iter-40100 loss: 0.1940 validation: 0.980000\n",
            "Iter-40200 loss: 0.1529 validation: 0.980000\n",
            "Iter-40300 loss: 0.1987 validation: 0.980000\n",
            "Iter-40400 loss: 0.2179 validation: 0.990000\n",
            "Iter-40500 loss: 0.2453 validation: 0.980000\n",
            "Iter-40600 loss: 0.1872 validation: 0.980000\n",
            "Iter-40700 loss: 0.1702 validation: 0.980000\n",
            "Iter-40800 loss: 0.1951 validation: 0.990000\n",
            "Iter-40900 loss: 0.1440 validation: 0.990000\n",
            "Iter-41000 loss: 0.1715 validation: 0.980000\n",
            "Iter-41100 loss: 0.1653 validation: 0.970000\n",
            "Iter-41200 loss: 0.1802 validation: 0.980000\n",
            "Iter-41300 loss: 0.2127 validation: 0.970000\n",
            "Iter-41400 loss: 0.1781 validation: 0.980000\n",
            "Iter-41500 loss: 0.2047 validation: 0.980000\n",
            "Iter-41600 loss: 0.1923 validation: 0.980000\n",
            "Iter-41700 loss: 0.1875 validation: 0.990000\n",
            "Iter-41800 loss: 0.2274 validation: 0.990000\n",
            "Iter-41900 loss: 0.1861 validation: 0.990000\n",
            "Iter-42000 loss: 0.2068 validation: 0.990000\n",
            "Iter-42100 loss: 0.1942 validation: 0.970000\n",
            "Iter-42200 loss: 0.2250 validation: 0.970000\n",
            "Iter-42300 loss: 0.1708 validation: 0.990000\n",
            "Iter-42400 loss: 0.1929 validation: 0.980000\n",
            "Iter-42500 loss: 0.2059 validation: 0.990000\n",
            "Iter-42600 loss: 0.2164 validation: 0.990000\n",
            "Iter-42700 loss: 0.1805 validation: 0.990000\n",
            "Iter-42800 loss: 0.1434 validation: 0.990000\n",
            "Iter-42900 loss: 0.1910 validation: 0.980000\n",
            "Iter-43000 loss: 0.1799 validation: 0.970000\n",
            "Iter-43100 loss: 0.2551 validation: 0.980000\n",
            "Iter-43200 loss: 0.2392 validation: 0.980000\n",
            "Iter-43300 loss: 0.2214 validation: 0.980000\n",
            "Iter-43400 loss: 0.2158 validation: 0.980000\n",
            "Iter-43500 loss: 0.1884 validation: 0.990000\n",
            "Iter-43600 loss: 0.1513 validation: 0.970000\n",
            "Iter-43700 loss: 0.1785 validation: 0.980000\n",
            "Iter-43800 loss: 0.2165 validation: 0.980000\n",
            "Iter-43900 loss: 0.1850 validation: 0.980000\n",
            "Iter-44000 loss: 0.2730 validation: 0.990000\n",
            "Iter-44100 loss: 0.1624 validation: 0.990000\n",
            "Iter-44200 loss: 0.1674 validation: 0.990000\n",
            "Iter-44300 loss: 0.1805 validation: 0.980000\n",
            "Iter-44400 loss: 0.2208 validation: 0.980000\n",
            "Iter-44500 loss: 0.1869 validation: 0.980000\n",
            "Iter-44600 loss: 0.2017 validation: 0.980000\n",
            "Iter-44700 loss: 0.1457 validation: 0.980000\n",
            "Iter-44800 loss: 0.1829 validation: 0.980000\n",
            "Iter-44900 loss: 0.1984 validation: 0.980000\n",
            "Iter-45000 loss: 0.2426 validation: 0.980000\n",
            "Iter-45100 loss: 0.1822 validation: 0.990000\n",
            "Iter-45200 loss: 0.1959 validation: 0.990000\n",
            "Iter-45300 loss: 0.1912 validation: 0.990000\n",
            "Iter-45400 loss: 0.2487 validation: 0.980000\n",
            "Iter-45500 loss: 0.1834 validation: 0.990000\n",
            "Iter-45600 loss: 0.2988 validation: 0.980000\n",
            "Iter-45700 loss: 0.1967 validation: 0.980000\n",
            "Iter-45800 loss: 0.2595 validation: 0.980000\n",
            "Iter-45900 loss: 0.1753 validation: 0.980000\n",
            "Iter-46000 loss: 0.1673 validation: 0.980000\n",
            "Iter-46100 loss: 0.2030 validation: 0.980000\n",
            "Iter-46200 loss: 0.2125 validation: 0.980000\n",
            "Iter-46300 loss: 0.1649 validation: 0.980000\n",
            "Iter-46400 loss: 0.1489 validation: 0.980000\n",
            "Iter-46500 loss: 0.1755 validation: 0.980000\n",
            "Iter-46600 loss: 0.1738 validation: 0.980000\n",
            "Iter-46700 loss: 0.1683 validation: 0.980000\n",
            "Iter-46800 loss: 0.1695 validation: 0.980000\n",
            "Iter-46900 loss: 0.1692 validation: 0.980000\n",
            "Iter-47000 loss: 0.1796 validation: 0.990000\n",
            "Iter-47100 loss: 0.1993 validation: 0.980000\n",
            "Iter-47200 loss: 0.1904 validation: 0.990000\n",
            "Iter-47300 loss: 0.2542 validation: 0.980000\n",
            "Iter-47400 loss: 0.1569 validation: 0.980000\n",
            "Iter-47500 loss: 0.1801 validation: 0.980000\n",
            "Iter-47600 loss: 0.1965 validation: 0.990000\n",
            "Iter-47700 loss: 0.1583 validation: 0.980000\n",
            "Iter-47800 loss: 0.1759 validation: 0.980000\n",
            "Iter-47900 loss: 0.1405 validation: 0.980000\n",
            "Iter-48000 loss: 0.1625 validation: 0.980000\n",
            "Iter-48100 loss: 0.2344 validation: 0.980000\n",
            "Iter-48200 loss: 0.1655 validation: 0.980000\n",
            "Iter-48300 loss: 0.2012 validation: 0.990000\n",
            "Iter-48400 loss: 0.2275 validation: 0.990000\n",
            "Iter-48500 loss: 0.2088 validation: 0.980000\n",
            "Iter-48600 loss: 0.1481 validation: 0.980000\n",
            "Iter-48700 loss: 0.1632 validation: 0.980000\n",
            "Iter-48800 loss: 0.1871 validation: 0.980000\n",
            "Iter-48900 loss: 0.2066 validation: 0.980000\n",
            "Iter-49000 loss: 0.1670 validation: 0.980000\n",
            "Iter-49100 loss: 0.1784 validation: 0.980000\n",
            "Iter-49200 loss: 0.1827 validation: 0.980000\n",
            "Iter-49300 loss: 0.1611 validation: 0.980000\n",
            "Iter-49400 loss: 0.2363 validation: 0.990000\n",
            "Iter-49500 loss: 0.1638 validation: 0.990000\n",
            "Iter-49600 loss: 0.1526 validation: 0.990000\n",
            "Iter-49700 loss: 0.1960 validation: 0.990000\n",
            "Iter-49800 loss: 0.1479 validation: 0.990000\n",
            "Iter-49900 loss: 0.2033 validation: 0.990000\n",
            "Iter-50000 loss: 0.1851 validation: 0.990000\n",
            "\n",
            "Best max_norm:  4.0 best dropout rate for input:  0.85 best dropout rate for hidden layers:  0.495 and result accuracy is:  0.9811\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1aeUQ0gJpTSM"
      },
      "source": [
        "Dropout NN + RELU + 3 layers + reLU + 1024 hidden units (50,000 weight update)\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gHK2ue7ppF6k"
      },
      "source": [
        "model = 'ff'\n",
        "n_iter = 50000\n",
        "alpha = 1e-3\n",
        "mb_size = 64\n",
        "n_experiment = 1\n",
        "reg = 1e-5\n",
        "print_after = 100\n",
        "p_dropout = [0.8, 0.5] #if nlayer=n, len(p_dropout) = n-1\n",
        "loss = 'cross_ent'\n",
        "nonlin = 'relu'\n",
        "solver = 'sgd'\n",
        "max_norm = 4\n",
        "nlayer = 3\n",
        "\n",
        "# Define default candidates\n",
        "candidates = np.array([[float(\"inf\"), float(\"inf\"), float(\"inf\")],[0.795, 0.8, 0.85], [0.495, 0.5, 0.55]]) # first hyper-param as max-norm, second as drop-out rate for input layer, third as drop-out rate for hidden layers\n",
        "\n",
        "run_mnist(1024, model, n_iter, alpha, mb_size, n_experiment, reg, print_after, p_dropout, loss, nonlin, solver, max_norm, x_train, x_validation, x_test, y_train, y_validation, y_test, candidates, nlayer)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rJFSNMHoqEVT"
      },
      "source": [
        "Dropout NN + Logistics + 3 layers + reLU + 1024 hidden units (50,000 weight update)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wK6wQMvXqN4K"
      },
      "source": [
        "model = 'ff'\n",
        "n_iter = 50000\n",
        "alpha = 1e-3\n",
        "mb_size = 64\n",
        "n_experiment = 1\n",
        "reg = 1e-5\n",
        "print_after = 100\n",
        "p_dropout = [0.8, 0.5] #if nlayer=n, len(p_dropout) = n-1\n",
        "loss = 'cross_ent'\n",
        "nonlin = 'sigmoid'\n",
        "solver = 'sgd'\n",
        "max_norm = 4\n",
        "nlayer = 3\n",
        "\n",
        "# Define default candidates\n",
        "candidates = np.array([[float(\"inf\"), float(\"inf\"), float(\"inf\")],[0.795, 0.8, 0.85], [0.495, 0.5, 0.55]]) # first hyper-param as max-norm, second as drop-out rate for input layer, third as drop-out rate for hidden layers\n",
        "\n",
        "run_mnist(1024, model, n_iter, alpha, mb_size, n_experiment, reg, print_after, p_dropout, loss, nonlin, solver, max_norm, x_train, x_validation, x_test, y_train, y_validation, y_test, candidates, nlayer)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rEVCme3etOWO"
      },
      "source": [
        "Dropout NN + max-norm constraint + ReLU + 2 layers + 4096 hidden units (50,000 weight updates)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xeGBOK8Y29Td",
        "outputId": "bf7154b9-0e2e-4e6c-e5d2-c7b96aa15530"
      },
      "source": [
        "model = 'ff'\n",
        "n_iter = 50000\n",
        "alpha = 1e-3\n",
        "mb_size = 64\n",
        "n_experiment = 1\n",
        "reg = 1e-5\n",
        "print_after = 100\n",
        "p_dropout = [0.85] #if nlayer=n, len(p_dropout) = n-1\n",
        "loss = 'cross_ent'\n",
        "nonlin = 'relu'\n",
        "solver = 'sgd'\n",
        "max_norm = 4\n",
        "nlayer = 2\n",
        "\n",
        "# Define default candidates\n",
        "candidates = np.array([[3, 3.5, 4],[0.795, 0.8, 0.85],[0.495, 0.5, 0.55]]) # first hyper-param as max-norm, second as drop-out rate for input layer, third as drop-out rate for hidden layers\n",
        "\n",
        "run_mnist(4096, model, n_iter, alpha, mb_size, n_experiment, reg, print_after, p_dropout, loss, nonlin, solver, max_norm, x_train, x_validation, x_test, y_train, y_validation, y_test, candidates, nlayer)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Experimenting on sgd\n",
            "\n",
            "Experiment-1\n",
            "Iter-100 loss: 1.8566 validation: 0.810000\n",
            "Iter-200 loss: 1.6109 validation: 0.830000\n",
            "Iter-300 loss: 1.4443 validation: 0.840000\n",
            "Iter-400 loss: 1.3656 validation: 0.850000\n",
            "Iter-500 loss: 1.3034 validation: 0.860000\n",
            "Iter-600 loss: 1.2665 validation: 0.860000\n",
            "Iter-700 loss: 1.2392 validation: 0.860000\n",
            "Iter-800 loss: 1.1759 validation: 0.860000\n",
            "Iter-900 loss: 1.1728 validation: 0.860000\n",
            "Iter-1000 loss: 1.1392 validation: 0.860000\n",
            "Iter-1100 loss: 1.0793 validation: 0.860000\n",
            "Iter-1200 loss: 1.1412 validation: 0.860000\n",
            "Iter-1300 loss: 1.0891 validation: 0.860000\n",
            "Iter-1400 loss: 1.0938 validation: 0.870000\n",
            "Iter-1500 loss: 0.9860 validation: 0.870000\n",
            "Iter-1600 loss: 0.9861 validation: 0.870000\n",
            "Iter-1700 loss: 1.0080 validation: 0.880000\n",
            "Iter-1800 loss: 0.9793 validation: 0.880000\n",
            "Iter-1900 loss: 1.0018 validation: 0.880000\n",
            "Iter-2000 loss: 0.8815 validation: 0.880000\n",
            "Iter-2100 loss: 1.0439 validation: 0.880000\n",
            "Iter-2200 loss: 1.0062 validation: 0.880000\n",
            "Iter-2300 loss: 0.9238 validation: 0.880000\n",
            "Iter-2400 loss: 1.0355 validation: 0.880000\n",
            "Iter-2500 loss: 0.9908 validation: 0.890000\n",
            "Iter-2600 loss: 0.9829 validation: 0.890000\n",
            "Iter-2700 loss: 1.0259 validation: 0.890000\n",
            "Iter-2800 loss: 0.8855 validation: 0.890000\n",
            "Iter-2900 loss: 0.9822 validation: 0.890000\n",
            "Iter-3000 loss: 0.9577 validation: 0.890000\n",
            "Iter-3100 loss: 0.8902 validation: 0.890000\n",
            "Iter-3200 loss: 0.8366 validation: 0.890000\n",
            "Iter-3300 loss: 0.9119 validation: 0.890000\n",
            "Iter-3400 loss: 0.9332 validation: 0.890000\n",
            "Iter-3500 loss: 0.8220 validation: 0.890000\n",
            "Iter-3600 loss: 0.8972 validation: 0.890000\n",
            "Iter-3700 loss: 0.7922 validation: 0.890000\n",
            "Iter-3800 loss: 0.9475 validation: 0.890000\n",
            "Iter-3900 loss: 0.8577 validation: 0.890000\n",
            "Iter-4000 loss: 0.7590 validation: 0.890000\n",
            "Iter-4100 loss: 0.8506 validation: 0.890000\n",
            "Iter-4200 loss: 0.9618 validation: 0.890000\n",
            "Iter-4300 loss: 0.8986 validation: 0.890000\n",
            "Iter-4400 loss: 0.7905 validation: 0.890000\n",
            "Iter-4500 loss: 0.7746 validation: 0.890000\n",
            "Iter-4600 loss: 0.7807 validation: 0.890000\n",
            "Iter-4700 loss: 0.8985 validation: 0.890000\n",
            "Iter-4800 loss: 0.9014 validation: 0.890000\n",
            "Iter-4900 loss: 0.9804 validation: 0.890000\n",
            "Iter-5000 loss: 0.7279 validation: 0.890000\n",
            "Iter-5100 loss: 0.7881 validation: 0.890000\n",
            "Iter-5200 loss: 0.7841 validation: 0.890000\n",
            "Iter-5300 loss: 0.7473 validation: 0.890000\n",
            "Iter-5400 loss: 0.8207 validation: 0.890000\n",
            "Iter-5500 loss: 0.7764 validation: 0.890000\n",
            "Iter-5600 loss: 0.6989 validation: 0.890000\n",
            "Iter-5700 loss: 0.8363 validation: 0.890000\n",
            "Iter-5800 loss: 0.8585 validation: 0.890000\n",
            "Iter-5900 loss: 0.7850 validation: 0.890000\n",
            "Iter-6000 loss: 0.8120 validation: 0.890000\n",
            "Iter-6100 loss: 0.8403 validation: 0.890000\n",
            "Iter-6200 loss: 0.7164 validation: 0.890000\n",
            "Iter-6300 loss: 0.8287 validation: 0.890000\n",
            "Iter-6400 loss: 0.8528 validation: 0.900000\n",
            "Iter-6500 loss: 0.7961 validation: 0.900000\n",
            "Iter-6600 loss: 0.6997 validation: 0.900000\n",
            "Iter-6700 loss: 0.8540 validation: 0.900000\n",
            "Iter-6800 loss: 0.7567 validation: 0.900000\n",
            "Iter-6900 loss: 0.8191 validation: 0.900000\n",
            "Iter-7000 loss: 0.7978 validation: 0.900000\n",
            "Iter-7100 loss: 0.6994 validation: 0.900000\n",
            "Iter-7200 loss: 0.8244 validation: 0.900000\n",
            "Iter-7300 loss: 0.8248 validation: 0.900000\n",
            "Iter-7400 loss: 0.7837 validation: 0.900000\n",
            "Iter-7500 loss: 0.7881 validation: 0.900000\n",
            "Iter-7600 loss: 0.7223 validation: 0.900000\n",
            "Iter-7700 loss: 0.8367 validation: 0.900000\n",
            "Iter-7800 loss: 0.7781 validation: 0.900000\n",
            "Iter-7900 loss: 0.7229 validation: 0.900000\n",
            "Iter-8000 loss: 0.7134 validation: 0.900000\n",
            "Iter-8100 loss: 0.7131 validation: 0.910000\n",
            "Iter-8200 loss: 0.8091 validation: 0.910000\n",
            "Iter-8300 loss: 0.8290 validation: 0.910000\n",
            "Iter-8400 loss: 0.7457 validation: 0.910000\n",
            "Iter-8500 loss: 0.7152 validation: 0.910000\n",
            "Iter-8600 loss: 0.7662 validation: 0.910000\n",
            "Iter-8700 loss: 0.8239 validation: 0.910000\n",
            "Iter-8800 loss: 0.7569 validation: 0.910000\n",
            "Iter-8900 loss: 0.6213 validation: 0.910000\n",
            "Iter-9000 loss: 0.6352 validation: 0.910000\n",
            "Iter-9100 loss: 0.6594 validation: 0.910000\n",
            "Iter-9200 loss: 0.6135 validation: 0.910000\n",
            "Iter-9300 loss: 0.6098 validation: 0.910000\n",
            "Iter-9400 loss: 0.7193 validation: 0.910000\n",
            "Iter-9500 loss: 0.6517 validation: 0.910000\n",
            "Iter-9600 loss: 0.7011 validation: 0.910000\n",
            "Iter-9700 loss: 0.7149 validation: 0.910000\n",
            "Iter-9800 loss: 0.7129 validation: 0.910000\n",
            "Iter-9900 loss: 0.7085 validation: 0.910000\n",
            "Iter-10000 loss: 0.6748 validation: 0.910000\n",
            "Iter-10100 loss: 0.6868 validation: 0.910000\n",
            "Iter-10200 loss: 0.6860 validation: 0.910000\n",
            "Iter-10300 loss: 0.8537 validation: 0.910000\n",
            "Iter-10400 loss: 0.6371 validation: 0.910000\n",
            "Iter-10500 loss: 0.6910 validation: 0.910000\n",
            "Iter-10600 loss: 0.7054 validation: 0.910000\n",
            "Iter-10700 loss: 0.6656 validation: 0.910000\n",
            "Iter-10800 loss: 0.6630 validation: 0.910000\n",
            "Iter-10900 loss: 0.6590 validation: 0.910000\n",
            "Iter-11000 loss: 0.7832 validation: 0.910000\n",
            "Iter-11100 loss: 0.7898 validation: 0.910000\n",
            "Iter-11200 loss: 0.6216 validation: 0.910000\n",
            "Iter-11300 loss: 0.6996 validation: 0.910000\n",
            "Iter-11400 loss: 0.6065 validation: 0.910000\n",
            "Iter-11500 loss: 0.7099 validation: 0.910000\n",
            "Iter-11600 loss: 0.5893 validation: 0.910000\n",
            "Iter-11700 loss: 0.6677 validation: 0.910000\n",
            "Iter-11800 loss: 0.6347 validation: 0.910000\n",
            "Iter-11900 loss: 0.7020 validation: 0.910000\n",
            "Iter-12000 loss: 0.7232 validation: 0.920000\n",
            "Iter-12100 loss: 0.7134 validation: 0.920000\n",
            "Iter-12200 loss: 0.7642 validation: 0.920000\n",
            "Iter-12300 loss: 0.6299 validation: 0.920000\n",
            "Iter-12400 loss: 0.7049 validation: 0.910000\n",
            "Iter-12500 loss: 0.7104 validation: 0.920000\n",
            "Iter-12600 loss: 0.8584 validation: 0.920000\n",
            "Iter-12700 loss: 0.7879 validation: 0.910000\n",
            "Iter-12800 loss: 0.6249 validation: 0.910000\n",
            "Iter-12900 loss: 0.5661 validation: 0.910000\n",
            "Iter-13000 loss: 0.6108 validation: 0.920000\n",
            "Iter-13100 loss: 0.6535 validation: 0.920000\n",
            "Iter-13200 loss: 0.7415 validation: 0.920000\n",
            "Iter-13300 loss: 0.7457 validation: 0.920000\n",
            "Iter-13400 loss: 0.7655 validation: 0.920000\n",
            "Iter-13500 loss: 0.5794 validation: 0.920000\n",
            "Iter-13600 loss: 0.6283 validation: 0.920000\n",
            "Iter-13700 loss: 0.7492 validation: 0.910000\n",
            "Iter-13800 loss: 0.6426 validation: 0.920000\n",
            "Iter-13900 loss: 0.5145 validation: 0.920000\n",
            "Iter-14000 loss: 0.6783 validation: 0.920000\n",
            "Iter-14100 loss: 0.6159 validation: 0.910000\n",
            "Iter-14200 loss: 0.6641 validation: 0.920000\n",
            "Iter-14300 loss: 0.6352 validation: 0.930000\n",
            "Iter-14400 loss: 0.5855 validation: 0.930000\n",
            "Iter-14500 loss: 0.5599 validation: 0.930000\n",
            "Iter-14600 loss: 0.6766 validation: 0.930000\n",
            "Iter-14700 loss: 0.6000 validation: 0.920000\n",
            "Iter-14800 loss: 0.7062 validation: 0.920000\n",
            "Iter-14900 loss: 0.6800 validation: 0.930000\n",
            "Iter-15000 loss: 0.6687 validation: 0.920000\n",
            "Iter-15100 loss: 0.6196 validation: 0.920000\n",
            "Iter-15200 loss: 0.5363 validation: 0.920000\n",
            "Iter-15300 loss: 0.6865 validation: 0.920000\n",
            "Iter-15400 loss: 0.6338 validation: 0.920000\n",
            "Iter-15500 loss: 0.7191 validation: 0.920000\n",
            "Iter-15600 loss: 0.6125 validation: 0.930000\n",
            "Iter-15700 loss: 0.6034 validation: 0.920000\n",
            "Iter-15800 loss: 0.6465 validation: 0.920000\n",
            "Iter-15900 loss: 0.7317 validation: 0.920000\n",
            "Iter-16000 loss: 0.6145 validation: 0.920000\n",
            "Iter-16100 loss: 0.6960 validation: 0.930000\n",
            "Iter-16200 loss: 0.6165 validation: 0.930000\n",
            "Iter-16300 loss: 0.5658 validation: 0.930000\n",
            "Iter-16400 loss: 0.6074 validation: 0.930000\n",
            "Iter-16500 loss: 0.6683 validation: 0.930000\n",
            "Iter-16600 loss: 0.6134 validation: 0.930000\n",
            "Iter-16700 loss: 0.6293 validation: 0.930000\n",
            "Iter-16800 loss: 0.6640 validation: 0.930000\n",
            "Iter-16900 loss: 0.6333 validation: 0.930000\n",
            "Iter-17000 loss: 0.6007 validation: 0.930000\n",
            "Iter-17100 loss: 0.6867 validation: 0.930000\n",
            "Iter-17200 loss: 0.6122 validation: 0.930000\n",
            "Iter-17300 loss: 0.6790 validation: 0.930000\n",
            "Iter-17400 loss: 0.5118 validation: 0.930000\n",
            "Iter-17500 loss: 0.6036 validation: 0.930000\n",
            "Iter-17600 loss: 0.8186 validation: 0.930000\n",
            "Iter-17700 loss: 0.6760 validation: 0.930000\n",
            "Iter-17800 loss: 0.7183 validation: 0.930000\n",
            "Iter-17900 loss: 0.6191 validation: 0.930000\n",
            "Iter-18000 loss: 0.5592 validation: 0.920000\n",
            "Iter-18100 loss: 0.7125 validation: 0.930000\n",
            "Iter-18200 loss: 0.5527 validation: 0.930000\n",
            "Iter-18300 loss: 0.6321 validation: 0.930000\n",
            "Iter-18400 loss: 0.6209 validation: 0.930000\n",
            "Iter-18500 loss: 0.5625 validation: 0.930000\n",
            "Iter-18600 loss: 0.6408 validation: 0.920000\n",
            "Iter-18700 loss: 0.6074 validation: 0.930000\n",
            "Iter-18800 loss: 0.6312 validation: 0.930000\n",
            "Iter-18900 loss: 0.6431 validation: 0.930000\n",
            "Iter-19000 loss: 0.7330 validation: 0.930000\n",
            "Iter-19100 loss: 0.5810 validation: 0.930000\n",
            "Iter-19200 loss: 0.7200 validation: 0.930000\n",
            "Iter-19300 loss: 0.5648 validation: 0.930000\n",
            "Iter-19400 loss: 0.6224 validation: 0.930000\n",
            "Iter-19500 loss: 0.6614 validation: 0.930000\n",
            "Iter-19600 loss: 0.6397 validation: 0.930000\n",
            "Iter-19700 loss: 0.6251 validation: 0.930000\n",
            "Iter-19800 loss: 0.6092 validation: 0.930000\n",
            "Iter-19900 loss: 0.5546 validation: 0.930000\n",
            "Iter-20000 loss: 0.5518 validation: 0.930000\n",
            "Iter-20100 loss: 0.5491 validation: 0.930000\n",
            "Iter-20200 loss: 0.6538 validation: 0.930000\n",
            "Iter-20300 loss: 0.5237 validation: 0.930000\n",
            "Iter-20400 loss: 0.6102 validation: 0.930000\n",
            "Iter-20500 loss: 0.5483 validation: 0.930000\n",
            "Iter-20600 loss: 0.5957 validation: 0.930000\n",
            "Iter-20700 loss: 0.5626 validation: 0.930000\n",
            "Iter-20800 loss: 0.7873 validation: 0.930000\n",
            "Iter-20900 loss: 0.6387 validation: 0.930000\n",
            "Iter-21000 loss: 0.7194 validation: 0.930000\n",
            "Iter-21100 loss: 0.5884 validation: 0.930000\n",
            "Iter-21200 loss: 0.6473 validation: 0.930000\n",
            "Iter-21300 loss: 0.6496 validation: 0.930000\n",
            "Iter-21400 loss: 0.5396 validation: 0.930000\n",
            "Iter-21500 loss: 0.5989 validation: 0.930000\n",
            "Iter-21600 loss: 0.6705 validation: 0.930000\n",
            "Iter-21700 loss: 0.6770 validation: 0.930000\n",
            "Iter-21800 loss: 0.7128 validation: 0.930000\n",
            "Iter-21900 loss: 0.5944 validation: 0.930000\n",
            "Iter-22000 loss: 0.5588 validation: 0.930000\n",
            "Iter-22100 loss: 0.5773 validation: 0.930000\n",
            "Iter-22200 loss: 0.7388 validation: 0.930000\n",
            "Iter-22300 loss: 0.5470 validation: 0.930000\n",
            "Iter-22400 loss: 0.5562 validation: 0.930000\n",
            "Iter-22500 loss: 0.5502 validation: 0.930000\n",
            "Iter-22600 loss: 0.5729 validation: 0.930000\n",
            "Iter-22700 loss: 0.6014 validation: 0.930000\n",
            "Iter-22800 loss: 0.5524 validation: 0.940000\n",
            "Iter-22900 loss: 0.6541 validation: 0.940000\n",
            "Iter-23000 loss: 0.5827 validation: 0.940000\n",
            "Iter-23100 loss: 0.6223 validation: 0.940000\n",
            "Iter-23200 loss: 0.6284 validation: 0.940000\n",
            "Iter-23300 loss: 0.5848 validation: 0.940000\n",
            "Iter-23400 loss: 0.5398 validation: 0.940000\n",
            "Iter-23500 loss: 0.5043 validation: 0.940000\n",
            "Iter-23600 loss: 0.5658 validation: 0.930000\n",
            "Iter-23700 loss: 0.6574 validation: 0.920000\n",
            "Iter-23800 loss: 0.4848 validation: 0.930000\n",
            "Iter-23900 loss: 0.5775 validation: 0.930000\n",
            "Iter-24000 loss: 0.5452 validation: 0.920000\n",
            "Iter-24100 loss: 0.5416 validation: 0.930000\n",
            "Iter-24200 loss: 0.6783 validation: 0.930000\n",
            "Iter-24300 loss: 0.5958 validation: 0.920000\n",
            "Iter-24400 loss: 0.6995 validation: 0.930000\n",
            "Iter-24500 loss: 0.6016 validation: 0.940000\n",
            "Iter-24600 loss: 0.5781 validation: 0.940000\n",
            "Iter-24700 loss: 0.6134 validation: 0.930000\n",
            "Iter-24800 loss: 0.7078 validation: 0.930000\n",
            "Iter-24900 loss: 0.5132 validation: 0.920000\n",
            "Iter-25000 loss: 0.5831 validation: 0.920000\n",
            "Iter-25100 loss: 0.7662 validation: 0.920000\n",
            "Iter-25200 loss: 0.7820 validation: 0.920000\n",
            "Iter-25300 loss: 0.6171 validation: 0.920000\n",
            "Iter-25400 loss: 0.5370 validation: 0.920000\n",
            "Iter-25500 loss: 0.6664 validation: 0.920000\n",
            "Iter-25600 loss: 0.6260 validation: 0.920000\n",
            "Iter-25700 loss: 0.5889 validation: 0.930000\n",
            "Iter-25800 loss: 0.7050 validation: 0.930000\n",
            "Iter-25900 loss: 0.8394 validation: 0.920000\n",
            "Iter-26000 loss: 0.6757 validation: 0.920000\n",
            "Iter-26100 loss: 0.7205 validation: 0.920000\n",
            "Iter-26200 loss: 0.5848 validation: 0.920000\n",
            "Iter-26300 loss: 0.6241 validation: 0.920000\n",
            "Iter-26400 loss: 0.5915 validation: 0.920000\n",
            "Iter-26500 loss: 0.7439 validation: 0.930000\n",
            "Iter-26600 loss: 0.6153 validation: 0.920000\n",
            "Iter-26700 loss: 0.4743 validation: 0.920000\n",
            "Iter-26800 loss: 0.5710 validation: 0.920000\n",
            "Iter-26900 loss: 0.6589 validation: 0.930000\n",
            "Iter-27000 loss: 0.6133 validation: 0.920000\n",
            "Iter-27100 loss: 0.5570 validation: 0.920000\n",
            "Iter-27200 loss: 0.6627 validation: 0.920000\n",
            "Iter-27300 loss: 0.5374 validation: 0.920000\n",
            "Iter-27400 loss: 0.6280 validation: 0.920000\n",
            "Iter-27500 loss: 0.6722 validation: 0.920000\n",
            "Iter-27600 loss: 0.6137 validation: 0.920000\n",
            "Iter-27700 loss: 0.6189 validation: 0.920000\n",
            "Iter-27800 loss: 0.6112 validation: 0.920000\n",
            "Iter-27900 loss: 0.5695 validation: 0.920000\n",
            "Iter-28000 loss: 0.5691 validation: 0.920000\n",
            "Iter-28100 loss: 0.6221 validation: 0.920000\n",
            "Iter-28200 loss: 0.5137 validation: 0.930000\n",
            "Iter-28300 loss: 0.5833 validation: 0.920000\n",
            "Iter-28400 loss: 0.5416 validation: 0.930000\n",
            "Iter-28500 loss: 0.5263 validation: 0.920000\n",
            "Iter-28600 loss: 0.5182 validation: 0.920000\n",
            "Iter-28700 loss: 0.6018 validation: 0.920000\n",
            "Iter-28800 loss: 0.6816 validation: 0.930000\n",
            "Iter-28900 loss: 0.5246 validation: 0.930000\n",
            "Iter-29000 loss: 0.5745 validation: 0.920000\n",
            "Iter-29100 loss: 0.5884 validation: 0.920000\n",
            "Iter-29200 loss: 0.5546 validation: 0.920000\n",
            "Iter-29300 loss: 0.5320 validation: 0.920000\n",
            "Iter-29400 loss: 0.5661 validation: 0.920000\n",
            "Iter-29500 loss: 0.6226 validation: 0.920000\n",
            "Iter-29600 loss: 0.4689 validation: 0.920000\n",
            "Iter-29700 loss: 0.4744 validation: 0.920000\n",
            "Iter-29800 loss: 0.6106 validation: 0.920000\n",
            "Iter-29900 loss: 0.5866 validation: 0.920000\n",
            "Iter-30000 loss: 0.6512 validation: 0.920000\n",
            "Iter-30100 loss: 0.6087 validation: 0.920000\n",
            "Iter-30200 loss: 0.4970 validation: 0.920000\n",
            "Iter-30300 loss: 0.7403 validation: 0.920000\n",
            "Iter-30400 loss: 0.4801 validation: 0.920000\n",
            "Iter-30500 loss: 0.5619 validation: 0.920000\n",
            "Iter-30600 loss: 0.4842 validation: 0.920000\n",
            "Iter-30700 loss: 0.6541 validation: 0.920000\n",
            "Iter-30800 loss: 0.5503 validation: 0.930000\n",
            "Iter-30900 loss: 0.5073 validation: 0.930000\n",
            "Iter-31000 loss: 0.6205 validation: 0.920000\n",
            "Iter-31100 loss: 0.4918 validation: 0.910000\n",
            "Iter-31200 loss: 0.4422 validation: 0.920000\n",
            "Iter-31300 loss: 0.8194 validation: 0.920000\n",
            "Iter-31400 loss: 0.5653 validation: 0.910000\n",
            "Iter-31500 loss: 0.5671 validation: 0.930000\n",
            "Iter-31600 loss: 0.4677 validation: 0.920000\n",
            "Iter-31700 loss: 0.6608 validation: 0.930000\n",
            "Iter-31800 loss: 0.5763 validation: 0.920000\n",
            "Iter-31900 loss: 0.5737 validation: 0.920000\n",
            "Iter-32000 loss: 0.6902 validation: 0.920000\n",
            "Iter-32100 loss: 0.5817 validation: 0.920000\n",
            "Iter-32200 loss: 0.5734 validation: 0.920000\n",
            "Iter-32300 loss: 0.5368 validation: 0.920000\n",
            "Iter-32400 loss: 0.5832 validation: 0.920000\n",
            "Iter-32500 loss: 0.6202 validation: 0.920000\n",
            "Iter-32600 loss: 0.5361 validation: 0.920000\n",
            "Iter-32700 loss: 0.4532 validation: 0.920000\n",
            "Iter-32800 loss: 0.5529 validation: 0.930000\n",
            "Iter-32900 loss: 0.5750 validation: 0.920000\n",
            "Iter-33000 loss: 0.6304 validation: 0.920000\n",
            "Iter-33100 loss: 0.5936 validation: 0.930000\n",
            "Iter-33200 loss: 0.5477 validation: 0.920000\n",
            "Iter-33300 loss: 0.5924 validation: 0.920000\n",
            "Iter-33400 loss: 0.5088 validation: 0.920000\n",
            "Iter-33500 loss: 0.5632 validation: 0.910000\n",
            "Iter-33600 loss: 0.5924 validation: 0.910000\n",
            "Iter-33700 loss: 0.5864 validation: 0.910000\n",
            "Iter-33800 loss: 0.6102 validation: 0.920000\n",
            "Iter-33900 loss: 0.5795 validation: 0.920000\n",
            "Iter-34000 loss: 0.6149 validation: 0.920000\n",
            "Iter-34100 loss: 0.4881 validation: 0.920000\n",
            "Iter-34200 loss: 0.5075 validation: 0.920000\n",
            "Iter-34300 loss: 0.6772 validation: 0.920000\n",
            "Iter-34400 loss: 0.5732 validation: 0.920000\n",
            "Iter-34500 loss: 0.5758 validation: 0.920000\n",
            "Iter-34600 loss: 0.6253 validation: 0.920000\n",
            "Iter-34700 loss: 0.5847 validation: 0.920000\n",
            "Iter-34800 loss: 0.4946 validation: 0.930000\n",
            "Iter-34900 loss: 0.5863 validation: 0.930000\n",
            "Iter-35000 loss: 0.4553 validation: 0.930000\n",
            "Iter-35100 loss: 0.5792 validation: 0.920000\n",
            "Iter-35200 loss: 0.5028 validation: 0.920000\n",
            "Iter-35300 loss: 0.6698 validation: 0.930000\n",
            "Iter-35400 loss: 0.6527 validation: 0.920000\n",
            "Iter-35500 loss: 0.6637 validation: 0.930000\n",
            "Iter-35600 loss: 0.5734 validation: 0.930000\n",
            "Iter-35700 loss: 0.6013 validation: 0.930000\n",
            "Iter-35800 loss: 0.5638 validation: 0.920000\n",
            "Iter-35900 loss: 0.4951 validation: 0.920000\n",
            "Iter-36000 loss: 0.5206 validation: 0.920000\n",
            "Iter-36100 loss: 0.5299 validation: 0.920000\n",
            "Iter-36200 loss: 0.6765 validation: 0.920000\n",
            "Iter-36300 loss: 0.5442 validation: 0.910000\n",
            "Iter-36400 loss: 0.7139 validation: 0.920000\n",
            "Iter-36500 loss: 0.4905 validation: 0.910000\n",
            "Iter-36600 loss: 0.5533 validation: 0.910000\n",
            "Iter-36700 loss: 0.5718 validation: 0.910000\n",
            "Iter-36800 loss: 0.5150 validation: 0.910000\n",
            "Iter-36900 loss: 0.4966 validation: 0.910000\n",
            "Iter-37000 loss: 0.5157 validation: 0.910000\n",
            "Iter-37100 loss: 0.6522 validation: 0.910000\n",
            "Iter-37200 loss: 0.6144 validation: 0.910000\n",
            "Iter-37300 loss: 0.5834 validation: 0.910000\n",
            "Iter-37400 loss: 0.5818 validation: 0.910000\n",
            "Iter-37500 loss: 0.7241 validation: 0.910000\n",
            "Iter-37600 loss: 0.6774 validation: 0.910000\n",
            "Iter-37700 loss: 0.5045 validation: 0.910000\n",
            "Iter-37800 loss: 0.5439 validation: 0.910000\n",
            "Iter-37900 loss: 0.5407 validation: 0.910000\n",
            "Iter-38000 loss: 0.5807 validation: 0.900000\n",
            "Iter-38100 loss: 0.6203 validation: 0.910000\n",
            "Iter-38200 loss: 0.6171 validation: 0.910000\n",
            "Iter-38300 loss: 0.4354 validation: 0.910000\n",
            "Iter-38400 loss: 0.5130 validation: 0.920000\n",
            "Iter-38500 loss: 0.7467 validation: 0.910000\n",
            "Iter-38600 loss: 0.4662 validation: 0.910000\n",
            "Iter-38700 loss: 0.5266 validation: 0.910000\n",
            "Iter-38800 loss: 0.5033 validation: 0.910000\n",
            "Iter-38900 loss: 0.6913 validation: 0.910000\n",
            "Iter-39000 loss: 0.5965 validation: 0.910000\n",
            "Iter-39100 loss: 0.5675 validation: 0.910000\n",
            "Iter-39200 loss: 0.6436 validation: 0.910000\n",
            "Iter-39300 loss: 0.5195 validation: 0.910000\n",
            "Iter-39400 loss: 0.6021 validation: 0.910000\n",
            "Iter-39500 loss: 0.6472 validation: 0.910000\n",
            "Iter-39600 loss: 0.4774 validation: 0.910000\n",
            "Iter-39700 loss: 0.5850 validation: 0.910000\n",
            "Iter-39800 loss: 0.4725 validation: 0.910000\n",
            "Iter-39900 loss: 0.4671 validation: 0.910000\n",
            "Iter-40000 loss: 0.5489 validation: 0.910000\n",
            "Iter-40100 loss: 0.6729 validation: 0.910000\n",
            "Iter-40200 loss: 0.5118 validation: 0.910000\n",
            "Iter-40300 loss: 0.5982 validation: 0.910000\n",
            "Iter-40400 loss: 0.5532 validation: 0.910000\n",
            "Iter-40500 loss: 0.5343 validation: 0.910000\n",
            "Iter-40600 loss: 0.5999 validation: 0.910000\n",
            "Iter-40700 loss: 0.4493 validation: 0.910000\n",
            "Iter-40800 loss: 0.5816 validation: 0.910000\n",
            "Iter-40900 loss: 0.6027 validation: 0.900000\n",
            "Iter-41000 loss: 0.6239 validation: 0.910000\n",
            "Iter-41100 loss: 0.5958 validation: 0.900000\n",
            "Iter-41200 loss: 0.5290 validation: 0.900000\n",
            "Iter-41300 loss: 0.6766 validation: 0.900000\n",
            "Iter-41400 loss: 0.5348 validation: 0.900000\n",
            "Iter-41500 loss: 0.4270 validation: 0.900000\n",
            "Iter-41600 loss: 0.6939 validation: 0.900000\n",
            "Iter-41700 loss: 0.5952 validation: 0.900000\n",
            "Iter-41800 loss: 0.5604 validation: 0.910000\n",
            "Iter-41900 loss: 0.6107 validation: 0.910000\n",
            "Iter-42000 loss: 0.5537 validation: 0.910000\n",
            "Iter-42100 loss: 0.4770 validation: 0.910000\n",
            "Iter-42200 loss: 0.6618 validation: 0.910000\n",
            "Iter-42300 loss: 0.4597 validation: 0.910000\n",
            "Iter-42400 loss: 0.6069 validation: 0.910000\n",
            "Iter-42500 loss: 0.5465 validation: 0.910000\n",
            "Iter-42600 loss: 0.5689 validation: 0.910000\n",
            "Iter-42700 loss: 0.4967 validation: 0.910000\n",
            "Iter-42800 loss: 0.6143 validation: 0.910000\n",
            "Iter-42900 loss: 0.5035 validation: 0.910000\n",
            "Iter-43000 loss: 0.4928 validation: 0.910000\n",
            "Iter-43100 loss: 0.5899 validation: 0.910000\n",
            "Iter-43200 loss: 0.5985 validation: 0.910000\n",
            "Iter-43300 loss: 0.5954 validation: 0.910000\n",
            "Iter-43400 loss: 0.5904 validation: 0.910000\n",
            "Iter-43500 loss: 0.4996 validation: 0.910000\n",
            "Iter-43600 loss: 0.5911 validation: 0.910000\n",
            "Iter-43700 loss: 0.5848 validation: 0.910000\n",
            "Iter-43800 loss: 0.7074 validation: 0.910000\n",
            "Iter-43900 loss: 0.6358 validation: 0.910000\n",
            "Iter-44000 loss: 0.7055 validation: 0.910000\n",
            "Iter-44100 loss: 0.4903 validation: 0.910000\n",
            "Iter-44200 loss: 0.6392 validation: 0.910000\n",
            "Iter-44300 loss: 0.6440 validation: 0.910000\n",
            "Iter-44400 loss: 0.6076 validation: 0.910000\n",
            "Iter-44500 loss: 0.6347 validation: 0.910000\n",
            "Iter-44600 loss: 0.5102 validation: 0.910000\n",
            "Iter-44700 loss: 0.4935 validation: 0.910000\n",
            "Iter-44800 loss: 0.5294 validation: 0.910000\n",
            "Iter-44900 loss: 0.5918 validation: 0.910000\n",
            "Iter-45000 loss: 0.6513 validation: 0.910000\n",
            "Iter-45100 loss: 0.5334 validation: 0.910000\n",
            "Iter-45200 loss: 0.5770 validation: 0.910000\n",
            "Iter-45300 loss: 0.6420 validation: 0.910000\n",
            "Iter-45400 loss: 0.4309 validation: 0.910000\n",
            "Iter-45500 loss: 0.7501 validation: 0.910000\n",
            "Iter-45600 loss: 0.5297 validation: 0.910000\n",
            "Iter-45700 loss: 0.6062 validation: 0.910000\n",
            "Iter-45800 loss: 0.6432 validation: 0.910000\n",
            "Iter-45900 loss: 0.5872 validation: 0.910000\n",
            "Iter-46000 loss: 0.5723 validation: 0.910000\n",
            "Iter-46100 loss: 0.5072 validation: 0.910000\n",
            "Iter-46200 loss: 0.5624 validation: 0.910000\n",
            "Iter-46300 loss: 0.4719 validation: 0.910000\n",
            "Iter-46400 loss: 0.6432 validation: 0.910000\n",
            "Iter-46500 loss: 0.5930 validation: 0.910000\n",
            "Iter-46600 loss: 0.5891 validation: 0.910000\n",
            "Iter-46700 loss: 0.6348 validation: 0.910000\n",
            "Iter-46800 loss: 0.5621 validation: 0.910000\n",
            "Iter-46900 loss: 0.6059 validation: 0.910000\n",
            "Iter-47000 loss: 0.6431 validation: 0.910000\n",
            "Iter-47100 loss: 0.5025 validation: 0.910000\n",
            "Iter-47200 loss: 0.4551 validation: 0.910000\n",
            "Iter-47300 loss: 0.6691 validation: 0.910000\n",
            "Iter-47400 loss: 0.6127 validation: 0.910000\n",
            "Iter-47500 loss: 0.5704 validation: 0.910000\n",
            "Iter-47600 loss: 0.6458 validation: 0.910000\n",
            "Iter-47700 loss: 0.5713 validation: 0.910000\n",
            "Iter-47800 loss: 0.6804 validation: 0.910000\n",
            "Iter-47900 loss: 0.4815 validation: 0.910000\n",
            "Iter-48000 loss: 0.6632 validation: 0.920000\n",
            "Iter-48100 loss: 0.6076 validation: 0.920000\n",
            "Iter-48200 loss: 0.7313 validation: 0.910000\n",
            "Iter-48300 loss: 0.5694 validation: 0.910000\n",
            "Iter-48400 loss: 0.6118 validation: 0.920000\n",
            "Iter-48500 loss: 0.4444 validation: 0.920000\n",
            "Iter-48600 loss: 0.6103 validation: 0.910000\n",
            "Iter-48700 loss: 0.4669 validation: 0.920000\n",
            "Iter-48800 loss: 0.5292 validation: 0.920000\n",
            "Iter-48900 loss: 0.4945 validation: 0.920000\n",
            "Iter-49000 loss: 0.5987 validation: 0.920000\n",
            "Iter-49100 loss: 0.5079 validation: 0.920000\n",
            "Iter-49200 loss: 0.5583 validation: 0.920000\n",
            "Iter-49300 loss: 0.5953 validation: 0.910000\n",
            "Iter-49400 loss: 0.6515 validation: 0.910000\n",
            "Iter-49500 loss: 0.4517 validation: 0.910000\n",
            "Iter-49600 loss: 0.5584 validation: 0.910000\n",
            "Iter-49700 loss: 0.5331 validation: 0.910000\n",
            "Iter-49800 loss: 0.5488 validation: 0.910000\n",
            "Iter-49900 loss: 0.4513 validation: 0.910000\n",
            "Iter-50000 loss: 0.5988 validation: 0.910000\n",
            "\n",
            "Best max_norm:  4.0 dropout rate for input:  1 best dropout rate for hidden layers:  0.5 and result accuracy is:  0.8235\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}