Neural_network_in_PyTorch.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Neural network in PyTorch.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/paulgureghian/Google_Colab_Notebooks/blob/master/Neural_network_in_PyTorch.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "metadata": {
        "id": "w6kBmzG7N_oW",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "** Created by Paul A. Gureghian on 11/21/18. **\n",
        "\n",
        "** This PyTorch notebook uses the MNIST dataset, **\n",
        "** from the Torchvision package, **\n",
        "** to build a deep neural network **"
      ]
    },
    {
      "metadata": {
        "id": "wN9EwJebNuu_",
        "colab_type": "code",
        "outputId": "c71bcb9c-b943-4a42-ec0b-c2bc22e11eaf",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        }
      },
      "cell_type": "code",
      "source": [
        "### install pytorch and torchvision and helper\n",
        "\n",
        "!pip3 install -U -q http://download.pytorch.org/whl/cu92/torch-0.4.1-cp36-cp36m-linux_x86_64.whl\n",
        "!pip3 install -U -q torchvision \n",
        "!pip3 install -U -q helper"
      ],
      "execution_count": 136,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tcmalloc: large alloc 1073750016 bytes == 0x57560000 @  0x7f7cdb0172a4 0x591a07 0x5b5d56 0x502e9a 0x506859 0x502209 0x502f3d 0x506859 0x504c28 0x502540 0x502f3d 0x506859 0x504c28 0x502540 0x502f3d 0x506859 0x504c28 0x502540 0x502f3d 0x507641 0x502209 0x502f3d 0x506859 0x504c28 0x502540 0x502f3d 0x507641 0x504c28 0x502540 0x502f3d 0x507641\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "jlCal2HlQIPa",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "### import packages\n",
        "\n",
        "import torch\n",
        "import helper\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "from collections import OrderedDict\n",
        "from torchvision import  datasets, transforms "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "jr9NzDDMSa0H",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "### define a transform to normalize the data\n",
        "\n",
        "transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5,),(0.5,)),])  "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "BdtXmfhnbifR",
        "colab_type": "code",
        "outputId": "b2ba600c-9d61-46b3-d713-2c7b1a294ab1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        }
      },
      "cell_type": "code",
      "source": [
        "### download and load the training data\n",
        "\n",
        "trainset = datasets.MNIST('~/.pytorch/MNIST_data/', download=True, train=True, transform=transform)\n",
        "print(\"trainset: \", trainset)\n",
        "print('')\n",
        "\n",
        "trainloader = torch.utils.data.DataLoader(trainset, batch_size=64, shuffle=True)\n",
        "print(\"trainloader: \", trainloader)"
      ],
      "execution_count": 139,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "trainset:  Dataset MNIST\n",
            "    Number of datapoints: 60000\n",
            "    Split: train\n",
            "    Root Location: /root/.pytorch/MNIST_data/\n",
            "    Transforms (if any): Compose(\n",
            "                             ToTensor()\n",
            "                             Normalize(mean=(0.5,), std=(0.5,))\n",
            "                         )\n",
            "    Target Transforms (if any): None\n",
            "\n",
            "trainloader:  <torch.utils.data.dataloader.DataLoader object at 0x7f164b82d588>\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "3owBafvZeZqx",
        "colab_type": "code",
        "outputId": "5164ad92-f844-4bbb-f7da-a8b107b0aaa2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        }
      },
      "cell_type": "code",
      "source": [
        "### create an iterator with 'trainloader'\n",
        "\n",
        "dataiter = iter(trainloader)\n",
        "images, labels = dataiter.next()\n",
        "\n",
        "print(\"datatype of images: \", (type(images))) \n",
        "print('')\n",
        "print(\"shape of images: \", images.shape)\n",
        "print('')\n",
        "print(\"shape of labels: \", labels.shape)   "
      ],
      "execution_count": 140,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "datatype of images:  <class 'torch.Tensor'>\n",
            "\n",
            "shape of images:  torch.Size([64, 1, 28, 28])\n",
            "\n",
            "shape of labels:  torch.Size([64])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "MAK-sS14jmBX",
        "colab_type": "code",
        "outputId": "3c12bac7-e505-400d-dd41-c1d7c944a206",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 347
        }
      },
      "cell_type": "code",
      "source": [
        "### plot an image \n",
        "\n",
        "plt.imshow(images[1].numpy().squeeze(), cmap='Greys_r');"
      ],
      "execution_count": 141,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUsAAAFKCAYAAACU6307AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAFMBJREFUeJzt3W1sVGX6x/Hf/FtKmSBbqC0rRl3D\nllhpu2Y3EIsBLBA2rDECGpEKROUFaCR08SENgaISKdRGI/CCUsGNEHR2uy8k8aENGiPRUhAjUnxo\ndV3SECytNghh6pY6+2Lznwic6VwznZkzZ/h+kibMPdecuW5O+XFmztxnfKFQKCQAwJD+z+0GAMAL\nCEsAMCAsAcCAsAQAA8ISAAwISwCwCKWAJMef48ePR7zPqz+ZOKdMnRdz8s5PquY1FF8qPmfp8/kc\nx0OhUMT7vCoT5yRl5ryYk3ekal5DxWF2vBvdtGmTjh07Jp/Pp7Vr16qsrCzeTQFA2osrLA8fPqyT\nJ08qEAjo22+/1dq1axUIBBLdGwCkjbhO8LS2tmrOnDmSpIkTJ+rs2bM6f/58QhsDgHQS15Flb2+v\nJk+eHL49btw49fT0aPTo0Y71x48fV0lJieN9KXjLNOUycU5SZs6LOXmH2/OK+z3LX4s2idLS0oiP\ny7Q3ozNxTlJmzos5eUc6nOCJ62V4YWGhent7w7fPnDmjgoKCeDYFAJ4QV1jecccdam5uliSdOHFC\nhYWFEV+CA0AmiOtl+B//+EdNnjxZDzzwgHw+nzZs2JDovgAgrfCh9ATLxDlJmTkv5uQdnn3PEgCu\nNoQlABgQlgBgQFgCgAFhCQAGhCUAGBCWAGBAWAKAAWEJAAaEJQAYEJYAYEBYAoABYQkABoQlABgQ\nlgBgQFgCgAFhCQAGhCUAGBCWAGBAWAKAAWEJAAZxfRUuvOWNN94w186cOTPifadPnw7/efv27eZt\nPv/88+ZaIF1xZAkABoQlABgQlgBgQFgCgAFhCQAGhCUAGBCWAGBAWAKAAWEJAAaEJQAYsNzRo+6/\n/35z7YIFC8y1I0aMiHjf+PHjHf8MXA04sgQAA8ISAAwISwAwICwBwICwBAADwhIADAhLADAgLAHA\ngLAEAANW8KQZv99vqtuzZ495mzk5OebawcFBx/GsrCz98ssv4dtHjhwxbxPIBBxZAoBBXEeWbW1t\nWr16tYqKiiRJkyZN0vr16xPaGACkk7hfhk+dOlVbt25NZC8AkLZ4GQ4ABnGH5TfffKOVK1dq8eLF\n+uijjxLZEwCkHV8oFArF+qDu7m4dPXpU8+bNU1dXl5YtW6aWlpaIZ13b29tVUlIy7GYBwC1xheXl\n7rvvPr300ku64YYbnJ/E53McD4VCEe/zquHOyfrRob6+PvM2E/XRoV/f9/DDD5u3GcvHnFKJ3z/v\nSNW8horDuF6G79+/X7t27ZIk9fT06IcffuDK2QAyWlxnw2fNmqUnn3xS7733ngYGBvTMM8/EdPQC\nAF4TV1iOHj1aO3bsSHQvAJC2WO6YZhYtWmSqG+qLxS4Xy9vStbW1juPr1q275L50fR8SSBY+ZwkA\nBoQlABgQlgBgQFgCgAFhCQAGhCUAGBCWAGBAWAKAAWEJAAaEJQAYJOQSbVGf5Cq/RFtubq758Z2d\nnaa666+/3rzNgYEBc+3UqVMdxz/77DPddttt4dvHjh0zbzNdXS2/f5nAs5doA4CrDWEJAAaEJQAY\nEJYAYEBYAoABYQkABoQlABgQlgBgQFgCgAFfWJYCM2fONNfGsjLH6uOPPzbXDrUyJxNW7QDx4sgS\nAAwISwAwICwBwICwBAADwhIADAhLADAgLAHAgLAEAAPCEgAMCEsAMGC541Xgtddec7uFjHPjjTea\na++66y5z7blz58y1Bw4ccBz/7W9/e8nt77//3rxNRMaRJQAYEJYAYEBYAoABYQkABoQlABgQlgBg\nQFgCgAFhCQAGhCUAGBCWAGDAcscUuO+++1x9/kOHDrn6/F5SX19vqlu5cqV5m36/P952htTb2+s4\n/vnnn19y+7HHHjNvs6mpaVg9ZTLTkWVHR4fmzJmjvXv3SpJOnz6tpUuXqrKyUqtXr9Z//vOfpDYJ\nAG6LGpYXLlzQxo0bVV5eHh7bunWrKisrtW/fPt100038bwQg40UNy5ycHDU2NqqwsDA81tbWptmz\nZ0uSKioq1NramrwOASANRH3PMjs7W9nZl5YFg0Hl5ORIkvLz89XT05Oc7gAgTQz7BE8oFIpac/z4\ncZWUlMT9eK9Jtzl98cUXCdlOus0rEbw8p4KCAtP4P/7xj1S0k3Ru76u4wtLv96u/v1+5ubnq7u6+\n5CW6k9LSUsfxUCgkn88XTwtpy2lOjY2N5scvX7480S1p8uTJ5tovv/zScfxq2VdePxteUFBwxSu9\nTDgbnqrfv6ECOa7PWU6bNk3Nzc2SpJaWFk2fPj2+zgDAI6IeWba3t2vLli06deqUsrOz1dzcrPr6\nelVXVysQCGjChAmaP39+KnoFANdEDcuSkhLt2bPnivFXX301KQ0BQDpiBU8K/OEPf0j4NmNZCBAM\nBhP+/F4yY8YM832PPvqoaZujRo0aVk+JYD3B8/e//928zcu/7GwoZ86cMddmAtaGA4ABYQkABoQl\nABgQlgBgQFgCgAFhCQAGhCUAGBCWAGBAWAKAAWEJAAa+UAouEhfp0kpXy2W/YvkrttZ+/fXX5m0W\nFxebayPx8r6K9Hc1adIkdXR0XDJWVFSU8Od/6aWXzLVnz5411z7zzDNXjPl8vmFd9/G6664z13Z3\nd8f9PLHy7CXaAOBqQ1gCgAFhCQAGhCUAGBCWAGBAWAKAAWEJAAaEJQAYEJYAYEBYAoAB3+6YAp2d\nnebaZCy3u9r9/ve/j+u+oezatctc+8QTT5hrH3jgAXNtpOV/l493dXWZt/nTTz+Za682HFkCgAFh\nCQAGhCUAGBCWAGBAWAKAAWEJAAaEJQAYEJYAYEBYAoABK3hS4F//+pe51rqiZOLEieZt3nzzzeba\n7777zlzrpuXLl5trh/qiq3i/BOuLL75IyPNfburUqeZapy/XcvrCsrfeesu8zWAwaK692nBkCQAG\nhCUAGBCWAGBAWAKAAWEJAAaEJQAYEJYAYEBYAoABYQkABoQlABiw3NGjsrPtu27kyJFJ7MQdv/nN\nb9xuwezZZ58111ZVVZlrf/755yvGcnNzrxjftGmTeZuIjCNLADAwhWVHR4fmzJmjvXv3SpKqq6t1\n9913a+nSpVq6dKk++OCDZPYIAK6L+lruwoUL2rhxo8rLyy8ZX7NmjSoqKpLWGACkk6hHljk5OWps\nbFRhYWEq+gGAtOQLOV0Uz8G2bds0duxYLVmyRNXV1erp6dHAwIDy8/O1fv16jRs3LuJj29vbVVJS\nkrCmASDV4jobfs899ygvL0/FxcXauXOntm/frpqamoj1paWljuOhUCjui6+mK6c5vfvuu+bHz507\nN9Et6dZbbzXXfvXVV47j6bav1qxZY66tr693HHe6UK7VE088Ya4dO3asuXbdunXm2khnw/v7+y8Z\nmzRpknmbXV1d5tpUStXv31C/D3GdDS8vL1dxcbEkadasWero6IivMwDwiLjCctWqVeH/gdra2lRU\nVJTQpgAg3UR9Gd7e3q4tW7bo1KlTys7OVnNzs5YsWaKqqiqNGjVKfr9ftbW1qegVAFwTNSxLSkq0\nZ8+eK8b//Oc/J6UhAEhHLHdMgV27dplrk/Gf0BtvvGGuXbVqVcT7pk+fHv7zwYMHh9XTcB05ciQp\n27WeRHj++efN28zJyTHXDg4OmmufeuqpK8a2bdt2xXi6nrTxGpY7AoABYQkABoQlABgQlgBgQFgC\ngAFhCQAGhCUAGBCWAGBAWAKAAWEJAAYsd0yBSNeIdOJ0jUInsSyhKysrM9e+8847pvucrhcQyT//\n+U9z7cmTJ011p06dMm8z0jUKh3M9y9zc3LgeF00wGDTXRtqvl48/8sgj5m0m45qRM2fONNcO9a2l\n+/btu+R2ZWVl3D3FgyNLADAgLAHAgLAEAAPCEgAMCEsAMCAsAcCAsAQAA8ISAAwISwAw8IXiXcIQ\ny5NEWBUQCoWSsmLATcOd0/z58011sXwJ2ciRI+NtJyEuXrxorv3ll18S/vyxrHZyWyz/HK2/Z8nY\nZiwuXLhgrn3zzTcdxxcvXqzXX3/9krFkrOAZ6u+KI0sAMCAsAcCAsAQAA8ISAAwISwAwICwBwICw\nBAADwhIADAhLADAgLAHAgOWOCZaqOf3pT38y127evNlcO3v2bMfx4Xy5VzLE8necjC8s++6778y1\nI0aMMNeOHTvWXPvpp59eMTZjxgx9+OGH5m1crqGhwVzb399vqnPqM5J///vfjuOp+nfFckcAGCbC\nEgAMCEsAMCAsAcCAsAQAA8ISAAwISwAwICwBwICwBAADwhIADFjumGDpOKdY+ikrK3Mc/+yzz3Tb\nbbeFby9cuHDYfbmtpqZGzz333CVjf/vb30yP7erqMj9PLH//WVlZ5lqn5Ybp+PuXCOmw3DHbsoG6\nujodPXpUFy9e1IoVK1RaWqqnn35ag4ODKigo0AsvvOCprxsFgFhFDctDhw6ps7NTgUBAfX19WrBg\ngcrLy1VZWal58+bpxRdfVFNTU1K+wxcA0kXU9yynTJmil19+WZI0ZswYBYNBtbW1ha9OU1FRodbW\n1uR2CQAuixqWWVlZ8vv9kqSmpibNmDFDwWAw/LI7Pz9fPT09ye0SAFxmes9Skg4cOKCmpibt3r1b\nc+fODY9bzg8dP35cJSUljvel0zUSEyUT5yT97yRPpqmpqRnythdl6u+f2/MyheXBgwe1Y8cOvfLK\nK7rmmmvk9/vV39+v3NxcdXd3q7CwcMjHl5aWOo5n4pm7dJwTZ8OdcTbcO9LhbHjUl+Hnzp1TXV2d\nGhoalJeXJ0maNm2ampubJUktLS2aPn16gloFgPQU9cjy7bffVl9fn6qqqsJjmzdv1rp16xQIBDRh\nwgTNnz8/qU0CgNuihuWiRYu0aNGiK8ZfffXVpDQEAOmIFTwJlolzkjJzXszJOzzxniUAgLAEABPC\nEgAMCEsAMCAsAcCAsAQAA8ISAAwISwAwICwBwICwBAADwhIADAhLADAgLAHAgLAEAAPCEgAMCEsA\nMCAsAcCAsAQAA8ISAAwISwAwICwBwICwBAADwhIADAhLADAgLAHAgLAEAAPCEgAMCEsAMCAsAcCA\nsAQAA8ISAAwISwAwICwBwICwBAADwhIADAhLADAgLAHAgLAEAAPCEgAMCEsAMCAsAcCAsAQAA8IS\nAAyyLUV1dXU6evSoLl68qBUrVuj999/XiRMnlJeXJ0lavny57rzzzmT2CQCuihqWhw4dUmdnpwKB\ngPr6+rRgwQLdfvvtWrNmjSoqKlLRIwC4LmpYTpkyRWVlZZKkMWPGKBgManBwMOmNAUA68YVCoZC1\nOBAI6JNPPlFWVpZ6eno0MDCg/Px8rV+/XuPGjYv8JD6f43goFIp4n1dl4pykzJwXc/KOVM1rqDg0\nh+WBAwfU0NCg3bt3q729XXl5eSouLtbOnTv1/fffq6amJuJj29vbVVJSEnvnAJAuQgYffvhh6N57\n7w319fVdcV9nZ2fowQcfHPLxkhx/hrrPqz+ZOKdMnRdz8s5PquY1lKgfHTp37pzq6urU0NAQPvu9\natUqdXV1SZLa2tpUVFQUbTMA4GlRT/C8/fbb6uvrU1VVVXhs4cKFqqqq0qhRo+T3+1VbW5vUJgHA\nbTGd4In7STjB43mZOC/m5B2pmtdQccgKHgAwICwBwICwBAADwhIADAhLADAgLAHAgLAEAAPCEgAM\nCEsAMCAsAcCAsAQAA8ISAAwISwAwICwBwICwBAADwhIADAhLADAgLAHAgLAEAAPCEgAMCEsAMCAs\nAcAgJV+FCwBex5ElABgQlgBgQFgCgAFhCQAGhCUAGBCWAGCQ7caTbtq0SceOHZPP59PatWtVVlbm\nRhsJ1dbWptWrV6uoqEiSNGnSJK1fv97lruLX0dGhxx57TA899JCWLFmi06dP6+mnn9bg4KAKCgr0\nwgsvKCcnx+02Y3L5nKqrq3XixAnl5eVJkpYvX64777zT3SZjVFdXp6NHj+rixYtasWKFSktLPb+f\npCvn9f7777u+r1IelocPH9bJkycVCAT07bffau3atQoEAqluIymmTp2qrVu3ut3GsF24cEEbN25U\neXl5eGzr1q2qrKzUvHnz9OKLL6qpqUmVlZUudhkbpzlJ0po1a1RRUeFSV8Nz6NAhdXZ2KhAIqK+v\nTwsWLFB5ebmn95PkPK/bb7/d9X2V8pfhra2tmjNnjiRp4sSJOnv2rM6fP5/qNjCEnJwcNTY2qrCw\nMDzW1tam2bNnS5IqKirU2trqVntxcZqT102ZMkUvv/yyJGnMmDEKBoOe30+S87wGBwdd7sqFsOzt\n7dXYsWPDt8eNG6eenp5Ut5EU33zzjVauXKnFixfro48+cruduGVnZys3N/eSsWAwGH45l5+f77l9\n5jQnSdq7d6+WLVumv/71r/rxxx9d6Cx+WVlZ8vv9kqSmpibNmDHD8/tJcp5XVlaW6/vKlfcsfy1T\nVlv+7ne/0+OPP6558+apq6tLy5YtU0tLiyffL4omU/bZPffco7y8PBUXF2vnzp3avn27ampq3G4r\nZgcOHFBTU5N2796tuXPnhse9vp9+Pa/29nbX91XKjywLCwvV29sbvn3mzBkVFBSkuo2EGz9+vP7y\nl7/I5/Ppxhtv1LXXXqvu7m6320oYv9+v/v5+SVJ3d3dGvJwtLy9XcXGxJGnWrFnq6OhwuaPYHTx4\nUDt27FBjY6OuueaajNlPl88rHfZVysPyjjvuUHNzsyTpxIkTKiws1OjRo1PdRsLt379fu3btkiT1\n9PTohx9+0Pjx413uKnGmTZsW3m8tLS2aPn26yx0N36pVq9TV1SXpf+/J/v8nGbzi3LlzqqurU0ND\nQ/gscSbsJ6d5pcO+cuWqQ/X19frkk0/k8/m0YcMG3XLLLaluIeHOnz+vJ598Uj/99JMGBgb0+OOP\na+bMmW63FZf29nZt2bJFp06dUnZ2tsaPH6/6+npVV1fr559/1oQJE1RbW6sRI0a43aqZ05yWLFmi\nnTt3atSoUfL7/aqtrVV+fr7brZoFAgFt27ZNN998c3hs8+bNWrdunWf3k+Q8r4ULF2rv3r2u7isu\n0QYABqzgAQADwhIADAhLADAgLAHAgLAEAAPCEgAMCEsAMCAsAcDgv9n11bhQ1PfWAAAAAElFTkSu\nQmCC\n",
            "text/plain": [
              "<matplotlib.figure.Figure at 0x7f164b638898>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "metadata": {
        "id": "DCqLg1-tlS5j",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "### define a sigmoid activation function\n",
        "\n",
        "def sigmoid(x):\n",
        "    return 1 / (1 + torch.exp(-x)) "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "seckXagsnNQC",
        "colab_type": "code",
        "outputId": "8694fe0d-c5b3-4bc1-9f43-681be2d039ed",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 170
        }
      },
      "cell_type": "code",
      "source": [
        "### flatten the input images \n",
        "\n",
        "inputs = images.view(images.shape[0], -1)\n",
        "print(\"inputs: \", inputs)  \n",
        "print('')\n",
        "print(\"shape of inputs: \", inputs.shape) "
      ],
      "execution_count": 143,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "inputs:  tensor([[-1., -1., -1.,  ..., -1., -1., -1.],\n",
            "        [-1., -1., -1.,  ..., -1., -1., -1.],\n",
            "        [-1., -1., -1.,  ..., -1., -1., -1.],\n",
            "        ...,\n",
            "        [-1., -1., -1.,  ..., -1., -1., -1.],\n",
            "        [-1., -1., -1.,  ..., -1., -1., -1.],\n",
            "        [-1., -1., -1.,  ..., -1., -1., -1.]])\n",
            "\n",
            "shape of inputs:  torch.Size([64, 784])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "uglidzqlrrz7",
        "colab_type": "code",
        "outputId": "d8cdd453-08c6-4eef-8b6b-253a0c1c2bfd",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 3417
        }
      },
      "cell_type": "code",
      "source": [
        "### create parameters\n",
        "\n",
        "w1 = torch.randn(784, 256)\n",
        "print(\"w1: \", w1) \n",
        "print(\"shape of w1: \", w1.shape)\n",
        "print('')\n",
        "\n",
        "b1 = torch.randn(256) \n",
        "print(\"b1: \", b1)\n",
        "print(\"shape of b1: \", b1.shape) \n",
        "print('')\n",
        "\n",
        "w2 = torch.randn(256, 10)\n",
        "print(\"w2: \", w2)\n",
        "print(\"shape of w2: \", w2.shape)\n",
        "print('')\n",
        "\n",
        "b2 = torch.randn(10)\n",
        "print(\"b2: \", b2)\n",
        "print(\"shape of b2: \", b2.shape)  \n",
        "print('')\n",
        "\n",
        "hidden_layer = sigmoid(torch.mm(inputs, w1) + b1)\n",
        "print(\"h: \", hidden_layer)\n",
        "print(\"shape of h: \", hidden_layer.shape)\n",
        "print('')\n",
        "\n",
        "output_layer = torch.mm(hidden_layer, w2) + b2\n",
        "print(\"output: \", output_layer)\n",
        "print(\"shape of output: \", output_layer.shape)  "
      ],
      "execution_count": 144,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "w1:  tensor([[ 0.5542,  1.1192,  1.3255,  ..., -0.5886, -0.5971,  0.7853],\n",
            "        [-1.1387, -0.2712,  1.2155,  ...,  0.7184,  0.4655, -0.0274],\n",
            "        [ 1.2081, -0.6354, -0.4101,  ..., -0.2509, -0.6757, -1.3447],\n",
            "        ...,\n",
            "        [-1.2878,  1.3005,  0.5847,  ...,  0.9735,  0.2870,  1.1535],\n",
            "        [-1.2289,  0.7048,  0.6144,  ..., -0.3104,  0.1162, -0.2544],\n",
            "        [ 0.1503, -0.1557, -0.8119,  ...,  1.5257, -0.4296,  0.9666]])\n",
            "shape of w1:  torch.Size([784, 256])\n",
            "\n",
            "b1:  tensor([ 1.7192,  1.2264, -0.2224, -0.1382,  0.9928,  1.2248, -1.0241, -0.0246,\n",
            "         0.4848, -0.3819,  0.7717, -0.4028, -1.7129, -0.3858, -0.3289,  1.1575,\n",
            "         1.5614,  0.3105,  0.4265, -0.4125,  0.4519,  0.8014, -1.1015, -1.9757,\n",
            "        -1.5486, -0.9962,  2.1693,  0.0048,  1.2996, -1.8837,  0.0698,  0.3138,\n",
            "         0.9798, -0.7421, -0.9639, -1.4858,  0.5091,  0.3485,  0.0302,  0.9235,\n",
            "         1.5319, -0.8577, -0.7936, -0.5507,  0.3849,  0.0980,  2.0498,  0.0309,\n",
            "         0.1728,  0.4624, -1.3288, -0.7939,  0.4878,  0.7743,  0.6990,  2.5631,\n",
            "        -0.0846, -0.0261,  1.8425,  0.6891,  1.2886, -0.5133,  1.3482,  1.1227,\n",
            "        -1.0702, -1.1665,  0.6696, -1.1693,  0.3144,  0.1100,  0.6791, -0.8123,\n",
            "        -0.5491,  0.8117,  0.8576,  0.5163, -0.4227, -0.2173,  0.9099,  0.1013,\n",
            "        -0.5789, -0.1824,  0.6768,  0.6705, -0.4644, -0.4267,  1.0736, -1.4531,\n",
            "         3.5048,  1.0382, -0.5553, -0.3949,  1.4734,  1.4793, -1.0748,  0.2505,\n",
            "         0.4600, -0.0480,  1.5170,  0.3830, -0.5316,  0.4660,  0.2162, -1.8739,\n",
            "        -0.2726,  1.8612,  1.3775,  0.8551, -1.2425,  0.5324,  1.6261, -1.7664,\n",
            "         2.5914,  1.6552,  1.4514,  0.7417, -0.1133, -0.0133, -0.5316,  0.2456,\n",
            "         0.1011,  0.1662, -0.8765, -0.1699, -1.5139, -0.0440,  0.2239, -0.8443,\n",
            "        -1.4177, -0.3280,  0.8555,  2.2165,  0.6595, -0.2560,  0.6532, -2.2349,\n",
            "         0.0941, -0.0585, -1.1707,  0.3049,  0.4460, -0.0844, -0.7207, -1.2251,\n",
            "        -0.4193, -0.5214,  0.5322, -0.6901, -0.2193, -0.8538,  0.7231, -0.3493,\n",
            "         0.5631,  0.6869, -0.7589,  1.5546, -0.2973, -1.2015, -1.9532,  2.6273,\n",
            "        -0.3111, -1.2107,  0.0503,  0.1819,  0.2469,  0.4558, -1.2700, -0.2237,\n",
            "         0.8982, -0.7043,  1.3959,  1.1486,  0.9301, -0.2966,  0.1166, -0.9039,\n",
            "         1.4921, -1.6271,  0.9252,  0.0077, -0.0284,  0.5117, -0.4882, -1.1560,\n",
            "        -0.6723,  1.6657,  0.1030,  0.4425, -0.3518,  0.0749,  0.4805, -0.4454,\n",
            "        -0.1855,  1.0303, -1.1625, -1.6769,  1.3403, -1.6753,  0.7507, -0.4310,\n",
            "        -0.4354,  2.6388, -2.1720,  1.2090, -0.8964, -0.2159,  0.4469, -0.2299,\n",
            "         0.6925,  0.7235,  0.6500, -2.1246, -0.8300,  0.7604,  0.4821,  1.0690,\n",
            "        -0.9338,  0.2542, -0.7094,  1.7111,  1.2126, -0.3855,  0.7790, -1.0355,\n",
            "        -2.3577,  0.3392, -1.1988, -1.1188,  0.8129, -1.4113,  1.9145, -1.8367,\n",
            "        -0.9980,  0.3168,  1.8516, -1.4792, -0.8679, -0.2559,  1.7923, -0.8107,\n",
            "        -1.0266,  0.0283,  1.7044,  2.3063,  0.5666,  1.2514,  1.0540,  0.9335,\n",
            "         0.9575,  1.0434, -0.2501,  0.9534,  1.3416,  0.6323,  0.8834,  1.0597])\n",
            "shape of b1:  torch.Size([256])\n",
            "\n",
            "w2:  tensor([[ 0.4067,  0.7323, -0.6151,  ...,  1.1317,  0.0385,  0.0457],\n",
            "        [-0.2166,  0.7100,  0.5113,  ...,  1.8393,  0.4055,  0.4488],\n",
            "        [-1.3356, -0.2827,  0.3057,  ...,  0.7174,  1.6998,  0.9829],\n",
            "        ...,\n",
            "        [ 1.1138, -0.1056, -0.6764,  ..., -0.7828,  0.3461,  0.4946],\n",
            "        [-0.9544,  0.0777, -0.6242,  ..., -1.0353,  1.0405, -1.2604],\n",
            "        [ 0.8695,  0.2272, -0.3728,  ...,  1.0257, -2.3088,  1.5736]])\n",
            "shape of w2:  torch.Size([256, 10])\n",
            "\n",
            "b2:  tensor([-1.6938, -0.2723,  0.4585,  0.7350,  0.5847, -0.1353,  0.6023,  0.1476,\n",
            "        -0.0630,  0.1231])\n",
            "shape of b2:  torch.Size([10])\n",
            "\n",
            "h:  tensor([[1.0000e+00, 1.0000e+00, 4.6004e-08,  ..., 9.9955e-01, 9.9989e-01,\n",
            "         1.0000e+00],\n",
            "        [1.1987e-04, 1.0000e+00, 2.2935e-08,  ..., 5.2211e-01, 2.7735e-10,\n",
            "         1.0000e+00],\n",
            "        [1.0946e-04, 1.9526e-03, 1.1578e-15,  ..., 1.0000e+00, 9.9199e-01,\n",
            "         1.0000e+00],\n",
            "        ...,\n",
            "        [9.9998e-01, 1.0000e+00, 7.5955e-12,  ..., 1.0000e+00, 1.0000e+00,\n",
            "         1.0000e+00],\n",
            "        [1.0000e+00, 1.0000e+00, 4.6593e-12,  ..., 1.0000e+00, 1.0000e+00,\n",
            "         1.0000e+00],\n",
            "        [1.0000e+00, 1.0000e+00, 1.2906e-06,  ..., 1.0000e+00, 1.6107e-04,\n",
            "         1.0000e+00]])\n",
            "shape of h:  torch.Size([64, 256])\n",
            "\n",
            "output:  tensor([[ 14.4047,  18.0144,  -3.3403,  13.8640,   9.9413,   7.2394,  10.0774,\n",
            "          23.0397,  -5.2614,  -7.6710],\n",
            "        [  8.4483,  21.1874,  20.0152,  13.5640,  20.5191,  -4.5798,   2.1595,\n",
            "          14.5459,  -5.0475, -10.5764],\n",
            "        [ 12.8949,  17.9038,  13.4893,   8.7963,  16.6278,   9.4921,  15.4089,\n",
            "          21.4330,  -7.8575,  -4.1534],\n",
            "        [ 10.8003,  30.2741,   6.0242,  14.2088,  14.5829,   2.5297,  13.0971,\n",
            "          23.7375,  -6.3897,  -3.4015],\n",
            "        [ 12.6637,  21.9976,   8.0658,   7.3430,   4.5338,  11.2309,   7.8720,\n",
            "          20.2242, -10.4718, -11.4110],\n",
            "        [ 13.6727,  19.3646,  12.1577,   7.2322,  12.8654,   3.0127,  13.0756,\n",
            "          23.2166,  -9.8010, -14.1787],\n",
            "        [ 15.3762,  20.9363,  12.4928,   2.8253,   5.8459,  -0.2913,   5.2607,\n",
            "          28.4247,  -3.3226, -13.1973],\n",
            "        [  9.7168,  14.3205,   3.7941,   8.8758,  11.0695,   7.6074,   7.0182,\n",
            "          29.4786,  -6.1961,  -8.9285],\n",
            "        [ 13.0277,  17.4457,   7.7709,   9.2886,  16.8846,   7.8166,  10.4730,\n",
            "          20.8904,  -1.3715,  -3.6765],\n",
            "        [ 13.9218,  24.8316,  13.3977,   8.7848,  13.7014,   9.1740,  12.0634,\n",
            "          30.8556,  -4.8900,  -2.5320],\n",
            "        [ 10.5113,  25.9953,  12.3358,   6.6835,  15.6970,   4.0147,   5.7653,\n",
            "          22.7819,  -7.3009,  -5.7360],\n",
            "        [ 14.2580,  31.5226,  11.2462,   7.8809,  23.0312,   5.4099,   9.4124,\n",
            "          25.4663,  -9.3016, -12.3365],\n",
            "        [ 15.3983,  17.4554,   6.5760,   7.1918,  10.8539,   7.5599,   2.7160,\n",
            "          21.9609,   4.7473,  -6.8671],\n",
            "        [ 22.5755,  18.2210,  12.1224,   8.5738,  10.6017,   4.4282,   3.5018,\n",
            "          31.7445,  -7.0098,  -9.2806],\n",
            "        [ 10.3328,  20.0544,   8.2648,   7.1903,   9.9191,   7.0994,   9.7481,\n",
            "          21.4625,  -2.1697, -12.4986],\n",
            "        [ 22.6581,  16.2055,   5.2262,  18.4356,  14.8921,   1.5094,   0.4764,\n",
            "          28.0445,   0.4896,  -3.1192],\n",
            "        [ 14.8593,  18.7066,   8.7791,   8.3392,  14.0904,  -0.1703,   7.3031,\n",
            "          28.0275,  -5.8358,  -5.0807],\n",
            "        [  5.7399,  23.9388,  15.7424,  14.1638,  16.3243,   4.3917,  11.7687,\n",
            "          17.2595,  -2.5579, -10.6229],\n",
            "        [ 14.6380,  21.2350,  14.5861,   5.9438,  11.5508,   3.2403,  10.7313,\n",
            "          23.7111,  -5.5307, -10.7977],\n",
            "        [  6.9000,  21.9203,  16.2848,   9.3337,   8.2242,  -1.9363,   8.1104,\n",
            "          30.5420,  -4.7615, -10.0256],\n",
            "        [  9.4667,  21.8753,   9.7592,  12.1203,   8.7330,   4.2205,   9.4847,\n",
            "          19.1894,  -5.0306,  -8.9483],\n",
            "        [ 18.4135,  28.0244,  12.7857,   0.0779,  15.0387,   4.6832,   1.8775,\n",
            "          27.2004, -20.2985, -10.5231],\n",
            "        [ 12.2845,  19.8669,   7.1082,   5.5075,   2.3252,   6.6115,  10.4089,\n",
            "          19.5349,   5.6284,  -4.2603],\n",
            "        [  6.0548,  27.0612,   7.8725,   4.8843,  21.5106,   5.4121,   4.4168,\n",
            "          24.8058,  -3.0732,  -7.5373],\n",
            "        [ 17.6084,  26.2819,  11.3934,  16.8595,  18.7764,  10.2010,   3.3504,\n",
            "          22.5362,  -2.0960, -10.7668],\n",
            "        [  7.3229,  15.8355,  13.0046,   6.8684,  10.4662,   4.5201,   0.3568,\n",
            "          26.8115,  -3.2863, -15.8374],\n",
            "        [  5.0980,   8.1858,  12.2462,  13.6086,  11.3773,  14.5794,   2.2467,\n",
            "          24.5341,  -2.9901,  -8.9746],\n",
            "        [ 10.9884,  18.6437,   8.4533,   9.1332,   9.2734,  10.4534,  16.7541,\n",
            "          23.4692,  -7.2964, -13.4643],\n",
            "        [ 17.9719,  31.8338,  16.8365,   6.3324,  12.0042,   0.7056,   8.3324,\n",
            "          26.5367,  -1.9821,  -2.0153],\n",
            "        [ 12.4143,  15.1946,   8.0512,  10.9716,  14.7451,   2.8963,  11.7790,\n",
            "          21.4716,  -0.5365,  -6.2214],\n",
            "        [ 15.0363,  17.3886,   3.9820,  13.5965,  20.3060,  -0.2052,  13.8153,\n",
            "          20.9643, -10.0686,  -8.4862],\n",
            "        [ 11.3795,   6.6632,  11.3512,   9.2121,   9.7079,  -1.8579,   5.1760,\n",
            "          19.6438,  -7.6766,  -9.3777],\n",
            "        [  8.8664,  21.9598,   5.9930,  13.7574,  14.7446,   5.1858,  12.6570,\n",
            "          32.7321,  -2.4050, -14.0564],\n",
            "        [ 17.8194,  22.9889,  19.4441,  14.7346,  12.8594,   0.8478,  12.7414,\n",
            "          23.4026,  -6.8785, -10.7261],\n",
            "        [ 17.6378,  23.0522,   8.4201,  21.1554,  11.9906,   7.3106,  10.5782,\n",
            "          26.1071,  -5.6932, -18.3161],\n",
            "        [  7.9067,  16.3891,  12.6937,   7.2472,   4.9557,   1.5247,   1.3162,\n",
            "          30.2815,  -4.0831,  -5.8283],\n",
            "        [ 25.0045,  18.7989,  17.3235,  14.4536,  11.9073,   5.9505,  -2.1512,\n",
            "          17.3984,  -6.8044, -10.3519],\n",
            "        [ 11.6617,  11.8338,   7.2174,   9.0762,   5.5651,   5.8387,   5.7539,\n",
            "          22.7171,  -4.8327,  -5.1990],\n",
            "        [ 16.4382,  10.6185,   1.0622,   6.5596,   9.1328,   5.3863,   6.7346,\n",
            "          22.7420,  -0.2711,  -8.5206],\n",
            "        [  7.1686,  21.7682,   5.5984,  17.4108,  10.4622,  -0.1207,  11.2191,\n",
            "          29.8974,  -4.3857,  -8.8206],\n",
            "        [ 20.5197,  28.1133,  15.0516,  12.1446,  14.8846,   5.3717,  -7.8359,\n",
            "          18.7076, -11.2270,  -2.9979],\n",
            "        [ 15.7994,  15.0815,   7.7430,  13.3483,  13.3599,   8.6472,   8.7000,\n",
            "          35.3275,  -4.4293,  -4.2464],\n",
            "        [ 16.4951,  17.3010,  15.6812,   7.7973,  10.5304,   8.4342,   7.0770,\n",
            "          29.9065, -14.2429, -16.0746],\n",
            "        [ 18.7043,  10.7437,  15.9988,  11.4342,   6.4028,   3.5544,   4.4780,\n",
            "          33.1411,   0.3431, -10.6954],\n",
            "        [ 21.8211,  28.2807,  24.1805,  10.4954,   7.0742,  -0.6642,  -2.0366,\n",
            "          34.2344, -10.7408, -10.7941],\n",
            "        [ 20.7122,  21.4413,  14.3174,  12.8309,   9.7417,  11.1447,  13.0473,\n",
            "          22.2003,  -4.2360,  -7.8368],\n",
            "        [ 14.2750,  18.0175,  14.7863,  11.7481,  21.0565,   2.2827,   1.8570,\n",
            "          15.7597,  -8.3662,  -3.3476],\n",
            "        [ 12.7733,  20.1076,  17.0134,   3.8629,  13.2439,   2.5953,  10.3961,\n",
            "          32.4303,   3.5254, -16.1227],\n",
            "        [ 12.7448,  22.1550,  11.9051,  13.2148,  12.6365,   2.1814,   7.2018,\n",
            "          16.4142,  -2.4180,  -5.3824],\n",
            "        [  8.7175,  20.1326,   9.0700,  12.5891,  10.1120,   5.0011,   2.6735,\n",
            "          27.4154,  -7.7425, -11.7607],\n",
            "        [ 14.2170,  21.1064,   1.4559,   8.0357,  16.3128,  10.8238,   6.6956,\n",
            "          29.4967, -11.2038,  -7.4496],\n",
            "        [ 16.3713,  15.7669,  16.9319,   3.1814,  16.9863,  -0.4195,  -2.2518,\n",
            "          27.5562,   4.6025,  -6.8842],\n",
            "        [ 13.9486,  16.0916,  16.6648,  12.5048,  16.2998,  -0.9305,   4.8694,\n",
            "          23.8462,  -7.2172, -11.4759],\n",
            "        [  8.9593,  24.0765,   4.5008,  15.7934,  12.3035,  12.2990,   6.8720,\n",
            "          23.7083, -11.0317, -11.6707],\n",
            "        [  6.5170,  22.6166,  13.1094,  10.9657,  12.1807,   4.1813,   3.3206,\n",
            "          21.6602,  -6.5117, -10.1211],\n",
            "        [ 12.3814,  16.4958,   1.7095,   9.8263,  17.3307,   7.1678,   4.2556,\n",
            "          27.0088,   0.1652,  -9.6537],\n",
            "        [  7.4714,  23.9656,  -0.1148,   2.7618,  10.2494,  10.9273,   5.8964,\n",
            "          25.1412, -10.2768, -11.5958],\n",
            "        [ 15.8060,  11.4160,   7.1604,  14.5531,   6.0312,   6.2174,   8.7103,\n",
            "          30.4658,  -4.7606, -10.5958],\n",
            "        [  6.8247,  23.8444,   8.5332,   3.8484,  11.6724,  10.9859,   7.1943,\n",
            "          21.6053,  -5.8170, -14.7175],\n",
            "        [  7.2907,  17.8146,   9.9649,  16.0370,   7.7927,  10.0447,  12.4643,\n",
            "          31.0067, -13.8131,  -6.7556],\n",
            "        [ 16.3539,  25.5463,  16.5124,  17.0718,  11.7907,   7.3362,  16.9184,\n",
            "          28.9630, -13.4200, -12.1364],\n",
            "        [ 13.6928,  31.9776,  10.2820,  18.0457,  15.1106,   7.9703,   9.1554,\n",
            "          24.6398,  -6.1767,  -9.6889],\n",
            "        [ 12.6373,  22.4409,  12.4737,   9.5732,  12.2751,  12.7181,  12.2746,\n",
            "          23.8831, -13.5090,  -6.7024],\n",
            "        [ 15.1421,  26.7021,  16.4825,  14.8954,  19.1899,   7.2600,  14.6705,\n",
            "          27.8787, -14.9503,  -5.6625]])\n",
            "shape of output:  torch.Size([64, 10])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "1UGf3ExSvWCn",
        "colab_type": "code",
        "outputId": "f7925a40-8d3a-49c3-bcd3-486b534ba097",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 2448
        }
      },
      "cell_type": "code",
      "source": [
        "### define a softmax activation function\n",
        "\n",
        "def softmax(x):\n",
        "  return torch.exp(x) / torch.sum(torch.exp(x), dim=1).view(-1, 1) \n",
        "\n",
        "probabilities = softmax(output_layer) \n",
        "print(\"sum of probabilities tensor:\\n \", probabilities.sum(dim=1))\n",
        "print('')\n",
        "print(\"probabilities tensor:\\n \", probabilities)\n",
        "print('')\n",
        "print(\"shape of probabilities tensor: \", probabilities.shape)\n",
        "print('')\n",
        "print(\"datatype of probabilities tensor: \", type(probabilities)) "
      ],
      "execution_count": 145,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "sum of probabilities tensor:\n",
            "  tensor([1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000,\n",
            "        1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000,\n",
            "        1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000,\n",
            "        1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000,\n",
            "        1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000,\n",
            "        1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000,\n",
            "        1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000,\n",
            "        1.0000])\n",
            "\n",
            "probabilities tensor:\n",
            "  tensor([[1.7655e-04, 6.5248e-03, 3.4700e-12, 1.0282e-04, 2.0345e-06, 1.3647e-07,\n",
            "         2.3311e-06, 9.9319e-01, 5.0820e-13, 4.5660e-14],\n",
            "        [1.6086e-06, 5.4822e-01, 1.6978e-01, 2.6802e-04, 2.8101e-01, 3.5351e-12,\n",
            "         2.9872e-09, 7.1545e-04, 2.2145e-12, 8.7925e-15],\n",
            "        [1.8824e-04, 2.8187e-02, 3.4106e-04, 3.1239e-06, 7.8684e-03, 6.2642e-06,\n",
            "         2.3255e-03, 9.6108e-01, 1.8282e-13, 7.4252e-12],\n",
            "        [3.4832e-09, 9.9855e-01, 2.9360e-11, 1.0527e-07, 1.5303e-07, 8.9150e-13,\n",
            "         3.4632e-08, 1.4473e-03, 1.1925e-16, 2.3670e-15],\n",
            "        [7.5544e-05, 8.5480e-01, 7.6092e-07, 3.6934e-07, 2.2254e-08, 1.8028e-05,\n",
            "         6.2687e-07, 1.4511e-01, 6.7697e-15, 2.6465e-15],\n",
            "        [7.0132e-05, 2.0792e-02, 1.5417e-05, 1.1190e-07, 3.1284e-05, 1.6458e-09,\n",
            "         3.8603e-05, 9.7905e-01, 4.4815e-15, 5.6264e-17],\n",
            "        [2.1520e-06, 5.5923e-04, 1.2039e-07, 7.6220e-12, 1.5627e-10, 3.3770e-13,\n",
            "         8.7047e-11, 9.9944e-01, 1.6295e-14, 8.3854e-19],\n",
            "        [2.6154e-09, 2.6118e-07, 7.0046e-12, 1.1280e-09, 1.0116e-08, 3.1730e-10,\n",
            "         1.7602e-10, 1.0000e+00, 3.2114e-16, 2.0893e-17],\n",
            "        [3.6630e-04, 3.0377e-02, 1.9092e-06, 8.7095e-06, 1.7333e-02, 1.9985e-06,\n",
            "         2.8467e-05, 9.5188e-01, 2.0433e-10, 2.0384e-11],\n",
            "        [4.4128e-08, 2.4141e-03, 2.6126e-08, 2.5926e-10, 3.5398e-08, 3.8260e-10,\n",
            "         6.8803e-09, 9.9759e-01, 2.9843e-16, 3.1544e-15],\n",
            "        [1.8124e-07, 9.6130e-01, 1.1236e-06, 3.9432e-09, 3.2385e-05, 2.7339e-10,\n",
            "         1.5743e-09, 3.8664e-02, 3.3304e-15, 1.5927e-14],\n",
            "        [3.1696e-08, 9.9746e-01, 1.5594e-09, 5.3881e-11, 2.0472e-04, 4.5530e-12,\n",
            "         2.4922e-10, 2.3372e-03, 1.8587e-18, 8.9361e-20],\n",
            "        [1.3949e-03, 1.0912e-02, 2.0561e-07, 3.8063e-07, 1.4822e-05, 5.5000e-07,\n",
            "         4.3319e-09, 9.8768e-01, 3.3026e-08, 2.9838e-13],\n",
            "        [1.0421e-04, 1.3391e-06, 3.0075e-09, 8.6510e-11, 6.5727e-10, 1.3698e-12,\n",
            "         5.4241e-13, 9.9989e-01, 1.4764e-17, 1.5240e-18],\n",
            "        [1.1787e-05, 1.9653e-01, 1.4903e-06, 5.0888e-07, 7.7936e-06, 4.6470e-07,\n",
            "         6.5685e-06, 8.0344e-01, 4.3817e-11, 1.4317e-15],\n",
            "        [4.5571e-03, 7.1843e-06, 1.2249e-10, 6.6819e-05, 1.9318e-06, 2.9780e-12,\n",
            "         1.0600e-12, 9.9537e-01, 1.0741e-12, 2.9090e-14],\n",
            "        [1.9103e-06, 8.9529e-05, 4.3700e-09, 2.8148e-09, 8.8543e-07, 5.6729e-13,\n",
            "         9.9883e-10, 9.9991e-01, 1.9649e-15, 4.1809e-15],\n",
            "        [1.2457e-08, 9.9792e-01, 2.7506e-04, 5.6735e-05, 4.9221e-04, 3.2352e-09,\n",
            "         5.1724e-06, 1.2540e-03, 3.1026e-12, 9.7531e-16],\n",
            "        [1.0579e-04, 7.7539e-02, 1.0044e-04, 1.7726e-08, 4.8274e-06, 1.1871e-09,\n",
            "         2.1271e-06, 9.2225e-01, 1.8421e-13, 9.5030e-16],\n",
            "        [5.3996e-11, 1.8012e-04, 6.4283e-07, 6.1561e-10, 2.0297e-10, 7.8483e-15,\n",
            "         1.8114e-10, 9.9982e-01, 4.6539e-16, 2.4080e-18],\n",
            "        [3.8226e-06, 9.3612e-01, 5.1213e-06, 5.4301e-05, 1.8352e-06, 2.0136e-08,\n",
            "         3.8918e-06, 6.3807e-02, 1.9330e-12, 3.8445e-14],\n",
            "        [4.6563e-05, 6.9505e-01, 1.6746e-07, 5.0701e-13, 1.5937e-06, 5.0708e-11,\n",
            "         3.0659e-12, 3.0490e-01, 7.1721e-22, 1.2620e-17],\n",
            "        [2.9644e-04, 5.8205e-01, 1.6746e-06, 3.3785e-07, 1.4018e-08, 1.0191e-06,\n",
            "         4.5436e-05, 4.1761e-01, 3.8129e-07, 1.9349e-11],\n",
            "        [6.7953e-10, 9.0195e-01, 4.1846e-09, 2.1081e-10, 3.5042e-03, 3.5737e-10,\n",
            "         1.3208e-10, 9.4551e-02, 7.3790e-14, 8.4970e-16],\n",
            "        [1.6697e-04, 9.7616e-01, 3.3383e-07, 7.8963e-05, 5.3691e-04, 1.0131e-07,\n",
            "         1.0728e-10, 2.3055e-02, 4.6256e-13, 7.9333e-17],\n",
            "        [3.4373e-09, 1.7107e-05, 1.0087e-06, 2.1819e-09, 7.9680e-08, 2.0843e-10,\n",
            "         3.2426e-12, 9.9998e-01, 8.4856e-14, 3.0049e-19],\n",
            "        [3.6225e-09, 7.9438e-08, 4.6071e-06, 1.7992e-05, 1.9322e-06, 4.7501e-05,\n",
            "         2.0926e-10, 9.9993e-01, 1.1127e-12, 2.8011e-15],\n",
            "        [3.7640e-06, 7.9491e-03, 2.9833e-07, 5.8881e-07, 6.7741e-07, 2.2044e-06,\n",
            "         1.2014e-03, 9.9084e-01, 4.3121e-14, 9.0360e-17],\n",
            "        [9.4993e-07, 9.9502e-01, 3.0521e-07, 8.3701e-12, 2.4319e-09, 3.0132e-14,\n",
            "         6.1844e-11, 4.9809e-03, 2.0500e-15, 1.9831e-15],\n",
            "        [1.1615e-04, 1.8727e-03, 1.4797e-06, 2.7445e-05, 1.1948e-03, 8.5388e-09,\n",
            "         6.1536e-05, 9.9673e-01, 2.7579e-10, 9.3679e-13],\n",
            "        [1.7189e-03, 1.8065e-02, 2.7190e-08, 4.0733e-04, 3.3408e-01, 4.1298e-10,\n",
            "         5.0695e-04, 6.4522e-01, 2.1494e-14, 1.0460e-13],\n",
            "        [2.5738e-04, 2.3032e-06, 2.5020e-04, 2.9464e-05, 4.8377e-05, 4.5884e-10,\n",
            "         5.2050e-07, 9.9941e-01, 1.3634e-12, 2.4880e-13],\n",
            "        [4.3179e-11, 2.0973e-05, 2.4398e-12, 5.7465e-09, 1.5422e-08, 1.0885e-12,\n",
            "         1.9121e-09, 9.9998e-01, 5.4971e-16, 4.7863e-21],\n",
            "        [2.2328e-03, 3.9257e-01, 1.1336e-02, 1.0213e-04, 1.5658e-05, 9.5100e-11,\n",
            "         1.3916e-05, 5.9373e-01, 4.1945e-14, 8.9470e-16],\n",
            "        [1.9899e-04, 4.4698e-02, 1.9754e-08, 6.7066e-03, 7.0189e-07, 6.5128e-09,\n",
            "         1.7095e-07, 9.4840e-01, 1.4666e-14, 4.8331e-20],\n",
            "        [1.9176e-10, 9.2606e-07, 2.3001e-08, 9.9163e-11, 1.0027e-11, 3.2440e-13,\n",
            "         2.6336e-13, 1.0000e+00, 1.1904e-15, 2.0784e-16],\n",
            "        [9.9700e-01, 2.0120e-03, 4.6014e-04, 2.6091e-05, 2.0448e-06, 5.2922e-09,\n",
            "         1.6038e-12, 4.9593e-04, 1.5285e-14, 4.4014e-16],\n",
            "        [1.5800e-05, 1.8767e-05, 1.8559e-07, 1.1907e-06, 3.5559e-08, 4.6747e-08,\n",
            "         4.2947e-08, 9.9996e-01, 1.0846e-12, 7.5192e-13],\n",
            "        [1.8260e-03, 5.4206e-06, 3.8353e-10, 9.3607e-08, 1.2269e-06, 2.8955e-08,\n",
            "         1.1151e-07, 9.9817e-01, 1.0111e-10, 2.6428e-14],\n",
            "        [1.3455e-10, 2.9472e-04, 2.7988e-11, 3.7757e-06, 3.6246e-09, 9.1868e-14,\n",
            "         7.7265e-09, 9.9970e-01, 1.2910e-15, 1.5306e-17],\n",
            "        [5.0334e-04, 9.9941e-01, 2.1237e-06, 1.1604e-07, 1.7972e-06, 1.3279e-10,\n",
            "         2.4389e-16, 8.2199e-05, 8.2125e-18, 3.0783e-14],\n",
            "        [3.3039e-09, 1.6116e-09, 1.0476e-12, 2.8482e-10, 2.8814e-10, 2.5876e-12,\n",
            "         2.7279e-12, 1.0000e+00, 5.4178e-18, 6.5051e-18],\n",
            "        [1.4980e-06, 3.3535e-06, 6.6378e-07, 2.5008e-10, 3.8465e-09, 4.7282e-10,\n",
            "         1.2169e-10, 9.9999e-01, 6.7014e-20, 1.0732e-20],\n",
            "        [5.3723e-07, 1.8746e-10, 3.5907e-08, 3.7393e-10, 2.4417e-12, 1.4147e-13,\n",
            "         3.5626e-13, 1.0000e+00, 5.7014e-15, 9.1632e-20],\n",
            "        [4.0534e-06, 2.5893e-03, 4.2904e-05, 4.8879e-11, 1.5970e-12, 6.9594e-16,\n",
            "         1.7642e-16, 9.9736e-01, 2.9265e-20, 2.7748e-20],\n",
            "        [1.3325e-01, 2.7626e-01, 2.2257e-04, 5.0337e-05, 2.2922e-06, 9.3229e-06,\n",
            "         6.2495e-05, 5.9014e-01, 1.9491e-12, 5.3210e-14],\n",
            "        [1.0744e-03, 4.5345e-02, 1.7917e-03, 8.5856e-05, 9.4696e-01, 6.6526e-09,\n",
            "         4.3463e-09, 4.7423e-03, 1.5785e-13, 2.3867e-11],\n",
            "        [2.9044e-09, 4.4493e-06, 2.0161e-07, 3.9202e-13, 4.6498e-09, 1.1036e-13,\n",
            "         2.6955e-10, 1.0000e+00, 2.7973e-13, 8.1973e-22],\n",
            "        [8.1592e-05, 9.9648e-01, 3.5236e-05, 1.3055e-04, 7.3220e-05, 2.1087e-09,\n",
            "         3.1943e-07, 3.2008e-03, 2.1211e-11, 1.0942e-12],\n",
            "        [7.5733e-09, 6.8678e-04, 1.0774e-08, 3.6367e-07, 3.0544e-08, 1.8419e-10,\n",
            "         1.7965e-11, 9.9931e-01, 5.3803e-16, 9.6769e-18],\n",
            "        [2.3122e-07, 2.2702e-04, 6.6364e-13, 4.7810e-10, 1.8802e-06, 7.7688e-09,\n",
            "         1.2518e-10, 9.9977e-01, 2.1081e-18, 9.0020e-17],\n",
            "        [1.3880e-05, 7.5846e-06, 2.4314e-05, 2.5948e-11, 2.5674e-05, 7.0835e-13,\n",
            "         1.1338e-13, 9.9993e-01, 1.0747e-10, 1.1033e-15],\n",
            "        [5.0206e-05, 4.2804e-04, 7.5930e-04, 1.1851e-05, 5.2710e-04, 1.7333e-11,\n",
            "         5.7246e-09, 9.9822e-01, 3.2254e-14, 4.5610e-16],\n",
            "        [1.6078e-07, 5.9093e-01, 1.8618e-09, 1.4936e-04, 4.5563e-06, 4.5357e-06,\n",
            "         1.9941e-08, 4.0891e-01, 3.3439e-16, 1.7650e-16],\n",
            "        [7.3586e-08, 7.2235e-01, 5.3683e-05, 6.2926e-06, 2.1208e-05, 7.1188e-09,\n",
            "         3.0103e-09, 2.7757e-01, 1.6162e-13, 4.3746e-15],\n",
            "        [4.4398e-07, 2.7179e-05, 1.0294e-11, 3.4491e-08, 6.2630e-05, 2.4161e-09,\n",
            "         1.3133e-10, 9.9991e-01, 2.1976e-12, 1.1957e-16],\n",
            "        [1.6192e-08, 2.3584e-01, 8.2159e-12, 1.4586e-10, 2.6046e-07, 5.1303e-07,\n",
            "         3.3518e-09, 7.6416e-01, 3.1719e-16, 8.4825e-17],\n",
            "        [4.2985e-07, 5.3306e-09, 7.5608e-11, 1.2280e-07, 2.4444e-11, 2.9447e-11,\n",
            "         3.5620e-10, 1.0000e+00, 5.0276e-16, 1.4696e-18],\n",
            "        [3.6682e-08, 9.0371e-01, 2.0250e-07, 1.8700e-09, 4.6751e-06, 2.3531e-06,\n",
            "         5.3084e-08, 9.6287e-02, 1.1864e-13, 1.6173e-17],\n",
            "        [5.0151e-11, 1.8652e-06, 7.2720e-10, 3.1533e-07, 8.2853e-11, 7.8759e-10,\n",
            "         8.8543e-09, 1.0000e+00, 3.4279e-20, 3.9815e-17],\n",
            "        [3.2351e-06, 3.1778e-02, 3.7910e-06, 6.6326e-06, 3.3740e-08, 3.9224e-10,\n",
            "         5.6894e-06, 9.6820e-01, 3.7955e-19, 1.3700e-18],\n",
            "        [1.1447e-08, 9.9935e-01, 3.7795e-10, 8.8956e-07, 4.7257e-08, 3.7452e-11,\n",
            "         1.2251e-10, 6.5004e-04, 2.6885e-17, 8.0199e-19],\n",
            "        [1.0564e-05, 1.9119e-01, 8.9696e-06, 4.9328e-07, 7.3540e-06, 1.1453e-05,\n",
            "         7.3500e-06, 8.0876e-01, 4.6627e-17, 4.2139e-14],\n",
            "        [2.2480e-06, 2.3563e-01, 8.5880e-06, 1.7564e-06, 1.2873e-04, 8.4844e-10,\n",
            "         1.4027e-06, 7.6423e-01, 1.9179e-19, 2.0723e-15]])\n",
            "\n",
            "shape of probabilities tensor:  torch.Size([64, 10])\n",
            "\n",
            "datatype of probabilities tensor:  <class 'torch.Tensor'>\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "0EwEdbCTjjvE",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "**This section builds the same network as above,**\n",
        "\n",
        "**But it uses the PyTorch 'nn' module for simplicity.**"
      ]
    },
    {
      "metadata": {
        "id": "g28YNcDg190G",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "### build a network with torch.nn\n",
        "\n",
        "### import torch.nn\n",
        "\n",
        "from torch import  nn\n",
        "import torch.nn.functional as F "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "IvtIkKdFQFEk",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "### define a class named 'Network'\n",
        "\n",
        "class Network(nn.Module):\n",
        "    def __init__(self):\n",
        "       super().__init__()\n",
        "    \n",
        "       # inputs to hidden layer\n",
        "       self.hidden1 = nn.Linear(784, 128)\n",
        "       self.hidden2 = nn.Linear(128, 64) \n",
        "       # output layer \n",
        "       self.output = nn.Linear(64, 10) \n",
        "    \n",
        "       # define sigmoid activation function and softmax output function\n",
        "       self.relu = nn.ReLU()  \n",
        "       self.sigmoid = nn.Sigmoid() \n",
        "       self.softmax = nn.Softmax(dim=1) \n",
        "        \n",
        "        \n",
        "    ### pass the input tensor (x) through each operation\n",
        "    def forward(self, x):\n",
        "        # hidden layer with sigmoid activtion function \n",
        "        x = F.relu(self.hidden1(x))\n",
        "        x = F.relu(self.hidden2(x))\n",
        "        # output layer with softmax activation function\n",
        "        x = F.softmax(self.output(x), dim=1) \n",
        "        \n",
        "        return x  "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "JQHuxezTYMjv",
        "colab_type": "code",
        "outputId": "3e18e30f-4b64-4a12-b6fd-e0b74934b2f3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 153
        }
      },
      "cell_type": "code",
      "source": [
        "### create a network object \n",
        "\n",
        "model = Network() \n",
        "model "
      ],
      "execution_count": 148,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Network(\n",
              "  (hidden1): Linear(in_features=784, out_features=128, bias=True)\n",
              "  (hidden2): Linear(in_features=128, out_features=64, bias=True)\n",
              "  (output): Linear(in_features=64, out_features=10, bias=True)\n",
              "  (relu): ReLU()\n",
              "  (sigmoid): Sigmoid()\n",
              "  (softmax): Softmax()\n",
              ")"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 148
        }
      ]
    },
    {
      "metadata": {
        "id": "1f80eTLL2nDk",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "**Initializing weights and biases.**"
      ]
    },
    {
      "metadata": {
        "id": "Ae8V-W8o2vaO",
        "colab_type": "code",
        "outputId": "a6e58bfe-e9c9-4177-cd3d-7ff0b81feb0d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 510
        }
      },
      "cell_type": "code",
      "source": [
        "### print the weights and biases (automatically initialized)\n",
        "\n",
        "print(\"weights:\\n \", model.hidden1.weight)\n",
        "print(\"biases:\\n \", model.hidden1.bias)   "
      ],
      "execution_count": 149,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "weights:\n",
            "  Parameter containing:\n",
            "tensor([[-0.0241, -0.0169,  0.0115,  ...,  0.0023,  0.0098,  0.0296],\n",
            "        [ 0.0177,  0.0268,  0.0172,  ..., -0.0311,  0.0053,  0.0206],\n",
            "        [-0.0175, -0.0330,  0.0071,  ..., -0.0097,  0.0022, -0.0077],\n",
            "        ...,\n",
            "        [-0.0249,  0.0210, -0.0095,  ...,  0.0069, -0.0215, -0.0009],\n",
            "        [ 0.0262, -0.0184, -0.0054,  ..., -0.0190,  0.0010,  0.0185],\n",
            "        [-0.0298, -0.0039, -0.0310,  ...,  0.0327,  0.0222,  0.0332]],\n",
            "       requires_grad=True)\n",
            "biases:\n",
            "  Parameter containing:\n",
            "tensor([-0.0281,  0.0238, -0.0347, -0.0219,  0.0026,  0.0327, -0.0242, -0.0197,\n",
            "         0.0330, -0.0266, -0.0186,  0.0201,  0.0139,  0.0075, -0.0111, -0.0243,\n",
            "         0.0256,  0.0279,  0.0193,  0.0300, -0.0275, -0.0225, -0.0073,  0.0085,\n",
            "         0.0228,  0.0139,  0.0106,  0.0246,  0.0152, -0.0114,  0.0225, -0.0303,\n",
            "        -0.0254, -0.0142, -0.0044,  0.0087, -0.0184,  0.0220, -0.0308, -0.0000,\n",
            "         0.0043,  0.0026,  0.0245,  0.0199,  0.0182,  0.0037,  0.0043,  0.0318,\n",
            "         0.0063,  0.0054, -0.0217, -0.0008, -0.0097,  0.0179,  0.0021,  0.0202,\n",
            "        -0.0281,  0.0035,  0.0017,  0.0339, -0.0171, -0.0107, -0.0322,  0.0333,\n",
            "         0.0174, -0.0003, -0.0011, -0.0083, -0.0025,  0.0082,  0.0095, -0.0068,\n",
            "        -0.0351, -0.0127, -0.0337, -0.0048, -0.0164, -0.0039, -0.0084,  0.0070,\n",
            "         0.0098,  0.0170, -0.0184,  0.0332,  0.0031,  0.0106,  0.0201, -0.0044,\n",
            "         0.0194,  0.0109, -0.0016, -0.0283, -0.0219,  0.0292,  0.0242, -0.0241,\n",
            "         0.0084, -0.0330, -0.0341,  0.0146,  0.0223,  0.0342, -0.0311, -0.0000,\n",
            "        -0.0281,  0.0162,  0.0138,  0.0052,  0.0017, -0.0087, -0.0172, -0.0001,\n",
            "         0.0345,  0.0132,  0.0119,  0.0020, -0.0170, -0.0071, -0.0277, -0.0003,\n",
            "        -0.0214, -0.0195, -0.0154,  0.0121, -0.0174,  0.0136, -0.0048, -0.0046],\n",
            "       requires_grad=True)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "chquHnIH47gF",
        "colab_type": "code",
        "outputId": "9e5127d9-b11d-4676-8678-0251f6aeee21",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 136
        }
      },
      "cell_type": "code",
      "source": [
        "### print the weights (custom initialization)\n",
        "\n",
        "model.hidden1.weight.data.normal_(std=0.01) "
      ],
      "execution_count": 150,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[ 0.0161,  0.0056, -0.0059,  ..., -0.0100,  0.0120, -0.0049],\n",
              "        [-0.0066,  0.0051,  0.0017,  ...,  0.0089, -0.0283, -0.0221],\n",
              "        [ 0.0034, -0.0074, -0.0192,  ...,  0.0002, -0.0168,  0.0004],\n",
              "        ...,\n",
              "        [-0.0133,  0.0099, -0.0120,  ...,  0.0094, -0.0197,  0.0105],\n",
              "        [-0.0107, -0.0020, -0.0004,  ..., -0.0023, -0.0050,  0.0119],\n",
              "        [ 0.0127, -0.0039, -0.0034,  ..., -0.0182, -0.0016, -0.0067]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 150
        }
      ]
    },
    {
      "metadata": {
        "id": "D9THnjaK542P",
        "colab_type": "code",
        "outputId": "b8223459-3bfa-4f83-f505-cfe378197848",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 153
        }
      },
      "cell_type": "code",
      "source": [
        "### print the biases (custom initialization) \n",
        "\n",
        "model.hidden1.bias.data.fill_(0)  "
      ],
      "execution_count": 151,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
              "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
              "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
              "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
              "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
              "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
              "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
              "        0., 0.])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 151
        }
      ]
    },
    {
      "metadata": {
        "id": "PEWAeTiq6peM",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "**Forward pass.**\n",
        "\n",
        "**Pass an image into the network.**"
      ]
    },
    {
      "metadata": {
        "id": "lGD-0MEkYS8s",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "### define a function 'view_classify' \n",
        "### for viewing an image and its predicted classes\n",
        "\n",
        "def view_classify(img, ps, version=\"MNIST\"):\n",
        "    ''' Function for viewing an image and it's predicted classes.\n",
        "    '''\n",
        "    ps = ps.data.numpy().squeeze()\n",
        "\n",
        "    fig, (ax1, ax2) = plt.subplots(figsize=(6,9), ncols=2)\n",
        "    ax1.imshow(img.resize_(1, 28, 28).numpy().squeeze())\n",
        "    ax1.axis('off')\n",
        "    ax2.barh(np.arange(10), ps)\n",
        "    ax2.set_aspect(0.1)\n",
        "    ax2.set_yticks(np.arange(10))\n",
        "    if version == \"MNIST\":\n",
        "        ax2.set_yticklabels(np.arange(10))\n",
        "    elif version == \"Fashion\":\n",
        "        ax2.set_yticklabels(['T-shirt/top',\n",
        "                            'Trouser',\n",
        "                            'Pullover',\n",
        "                            'Dress',\n",
        "                            'Coat',\n",
        "                            'Sandal',\n",
        "                            'Shirt',\n",
        "                            'Sneaker',\n",
        "                            'Bag',\n",
        "                            'Ankle Boot'], size='small');\n",
        "    ax2.set_title('Class Probability')\n",
        "    ax2.set_xlim(0, 1.1)\n",
        "\n",
        "    plt.tight_layout()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "LcLTum_47Cbw",
        "colab_type": "code",
        "outputId": "134d33e4-61dd-4105-c37c-e02bbf5463ad",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 327
        }
      },
      "cell_type": "code",
      "source": [
        "### resize the image into a 1D vector\n",
        "\n",
        "images.resize_(64, 1, 784) \n",
        "\n",
        "### forward pass through the network\n",
        "\n",
        "img_idx = 0\n",
        "ps = model.forward(images[img_idx,:])\n",
        "print(\"ps: \", ps)\n",
        "print('')\n",
        "\n",
        "img = images[img_idx] \n",
        "print(\"shape of img: \", img.shape) \n",
        "print('')\n",
        "\n",
        "view_classify(img.view(1, 28, 28), ps)  "
      ],
      "execution_count": 153,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "ps:  tensor([[0.1064, 0.1046, 0.1090, 0.1001, 0.0940, 0.1052, 0.1003, 0.1067, 0.0899,\n",
            "         0.0839]], grad_fn=<SoftmaxBackward>)\n",
            "\n",
            "shape of img:  torch.Size([1, 784])\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaQAAADhCAYAAACHrsFeAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAGXpJREFUeJzt3XlUVdcd9vGH0QkloIJzoqbWRGvj\nEFedglIQHN68NVXBeZlBrdFGl0mcEEmtVl0a26qt1hiboqmmSpyqoGGpTY1DDFYXGuOwVqxTEBmc\nQBdczvtHWlZ9cy8RvZyzge/nL9iHffdjgvzY+x7Pz8eyLEsAADjM1+kAAABIFCQAgCEoSAAAI1CQ\nAABGoCABAIxAQQIAGIGCBKBCWJaldevWaeDAgYqJiVFUVJSSkpJ0+/ZtSdKMGTP0hz/8oUIzzJgx\nQz/5yU8UGxurmJgYxcbGatmyZXK5XOV6ncuXL+vZZ58t9/qjRo3Stm3bvjOelZWlgQMHSpKWL1+u\n2bNnS5LGjBmjU6dOSZI++uijcq9X2VGQAFSIJUuWaNeuXVq7dq3S0tK0fft2FRUVafz48bLznz+O\nHj1aqampSktL0+bNm/XZZ585/sM+PDxcO3fu/M74Bx98oHbt2snlcmnx4sUOJHMWBQmA1+Xn5ys5\nOVkLFy5UeHi4JKl27dpKTEzUq6+++p2CdPz4cb300kuKjY1V//799dlnn0mSiouLNXv2bMXExCg6\nOlqTJk3SnTt3PI5/n6CgIP3sZz/TwYMHJX27g1m2bJn69eunjIwM5efn64033lBMTIz69++vP/3p\nTw/MX7dunfr166fIyEh98sknkqSSkhK98847iomJUWRkpN566y0VFRWVzjl79qwGDx6siIgIJSQk\nyOVyedxxRUZG6tixYxo7dqxu376t2NhYbdiwQePHjy/9mpKSEnXv3l1ffvnlw/yvqFQoSAC87sSJ\nE2rUqJFat279wHiNGjUUGRkpX98Hf/QkJibqlVdeUWpqqsaNG6e5c+dKkv75z3/q8uXLSk1N1Z49\ne/T000/r+PHjHscfRnFxsQIDA0s/z8zM1N///nd16tRJ7777roKDg5WWlqYPP/xQf/3rX3Xs2DFJ\nksvlksvl0u7duzVv3jzNmTNHRUVF2rt3r44dO6adO3dq9+7dOnXqlHbt2lX6+keOHFFycrJSU1P1\n+eefa9++fd+bccGCBfLz81NqaqpiY2N1+PBh5eXlSZIyMjJUr149PfPMMw/1561MKEgAvC4/P1/1\n69d/6K/funWr+vXrJ0nq3LmzLl26JEkKDQ3VhQsXtHfvXhUWFmrKlCnq1auXx/Hvk5OToy1btig6\nOrp0LCIiorRAHjhwQMOHD5ckPfHEE4qOji7dTUnSoEGDJEk9evRQcXGx/v3vfysmJkZbtmxRQECA\natSooR/96Eel+SUpJiZGtWrVUq1atRQREaF//etfD/3fRZLq16+vLl26KC0tTZK0d+9e9e/fv1yv\nUVlQkAB4XUhIiLKysh7663fs2KHBgwcrJiZGL7/8cumRXocOHZSQkKDk5GT16NFD06ZN061btzyO\nu/OXv/xFsbGxio2N1auvvqohQ4aUFj9JCg4OLv04NzdX9erVK/28Xr16ysnJeeDP9V9169bVrVu3\nlJubq+nTp5feNJGenv7AkWRoaOh35pTXgAEDSt9zSk9PpyABwMN67rnnlJOTU3rH2H8VFRVp2bJl\nKiwsLB3LyspSQkKC5s+fr7S0NK1Zs+aBObGxsUpOTta+fftUWFiotWvXljn+//vvTQ2pqan6+OOP\nNWrUKI+5GzRooPz8/NLP8/Pz1aBBg9LPb968+cDHwcHBWrZsmfz9/bVjxw6lpqYqIiLigdd0N6e8\noqOjlZmZqQMHDqhWrVp6+umny/0alQEFCYDX1atXT6+++qqmT5+uixcvSpIKCwuVmJio06dPq1at\nWqVfm5ubq9q1a6tVq1YqLi7Wpk2bJEl3797Vli1btHLlSknfHqG1atVKkjyOP67evXuXrp+bm6u9\ne/eqd+/epdd37NghSTp48KBq1aqlFi1aKCcnR23atFFgYKDOnDmj48ePq6CgoHTOnj17dP/+fRUU\nFOjTTz9Vly5dvjdHQECASkpKSm/UqFu3rnr16qV33nnngd1dVePvdAAAVdPkyZMVHBysX/ziF3K5\nXPL19dVPf/pTJSUlPfB1bdu21QsvvKCYmBjVr19fM2bMUEZGhkaNGqX3339fs2bNUt++feXn56cn\nn3xSCxculCSP449jypQpSkpKUmxsrHx9fTVu3Dh16NBBly9fVu3atVVSUqKBAwfq3r17mj9/vvz9\n/fXyyy9r+vTpSklJUZcuXTR9+nTNnj1bHTp0kCR1795do0ePVlZWlnr37q1evXrp6tWrZeZo2LCh\nOnfurD59+mj16tXq1KmTBgwYoD179lTZ4zpJ8qEfEgCY7+TJk/rVr36lzZs3Ox2lwnBkBwCGKy4u\n1sqVK8t8/6sqoCABgMFOnz6t6OhohYWF6cUXX3Q6ToXiyA4AYAR2SAAAI3CXHWCw4mKX8vIKvv8L\nK1BISG0ykMFrGRo2rOvxGjskwGD+/n5ORyADGWzLQEECABiBggQAMAIFCQBgBAoSAMAIFCQAgBEo\nSAAAI1CQAABG4B/GAgb7P9O2PdK892dEejkJUPHYIQEAjEBBAgAYgSM7wEYlJSWaO3euzp07p4CA\nACUlJal169ZOxwKMwA4JsFF6erpu376tjRs3av78+Vq8eLHTkQBjUJAAG3399dfq0KGDJKlFixa6\nevWqXC6Xw6kAM3BkB9ioTZs2+uCDDzRmzBhdvHhRly5dUl5enho0aODVdcp6xL8Jr0cGMrhDQQJs\nFBERoYyMDI0YMUI//OEP1apVK1VE0+bs7Ntee62GDet69fXIUL0zlFXMKEiAzaZOnVr6cVRUlOrX\nr+9gGsAcvIcE2OjMmTOaOXOmJOkf//iHnn32Wfn68tcQkNghAbZq06aNLMvS4MGDVaNGDS1ZssTp\nSIAxKEiAjXx9fbVw4UKnYwBG4qwAAGAEdkiAwXYs/b+O31UF2IUdEgDACBQkAIAROLIDDPao/ZAk\neiKh8mGHBAAwAgUJAGAEjuwAG929e1fTp0/XzZs3VVRUpNdff129evVyOhZgBAoSYKOPP/5YLVu2\n1LRp05SVlaUxY8YoNTXV6ViAETiyA2wUEhKi/Px8SdKtW7cUEhLicCLAHOyQABsNGDBAKSkpio6O\n1q1bt7R69WqnIwHGoCABNtq2bZuaNGmitWvX6syZM5o1a5ZSUlIqZC1vNlGryk3hyGBOBgoSYKOM\njAz17NlTktS2bVtdv35dLpdLfn5+Xl/LW48cqgpN4chgToayihnvIQE2evLJJ3XixAlJ0pUrV1Sn\nTp0KKUZAZcQOCbBRXFycZs2apZEjR6q4uFhJSUlORwKMQUECbFSnTh397ne/czoGYCQKUiVUWFjo\n8dru3bvdjk+ePNnjnGvXrrkdtyyrfMEk1a3r+Xx43bp1bsd//vOfl3sdAFUPBQkwGP2QUJ1wUwMA\nwAgUJACAETiyAwz2OP2Q/he9kVAZsEMCABiBHZLBLly44HZ85MiRHuccPXq03OvUrFnT7XhcXJzH\nOefOnXM7fujQIY9zJk6c6HY8KirK45zg4GCP1wBULeyQAABGYIcE2Ohvf/ubtm/fXvp5Zmamjh8/\n7mAiwBwUJMBGQ4YM0ZAhQyR9e7zq6R8yA9URR3aAQ1auXOnxfTWgOmKHBDjg5MmTaty4sRo2bGjL\neo/bv6Yq9+AhgzkZKEiAAzZv3qxBgwbZtt7j9q9x+vFFZKg6GcoqZhQkm5SUlLgdP3DggMc5AwcO\ndDt+7949j3M89dZZuXKlxzmebu+uV6+exzmeMjRo0MDjHE8PXq2O/YCOHDmihIQEp2MARuE9JMBm\nWVlZqlOnjgIDA52OAhiFggTYLDs7W6GhoU7HAIxDQQJs1r59e7333ntOxwCMQ0ECABiBmxoAg9Gg\nD9UJBckm7777rtvx6dOnl/u1unXr5vHatm3u2xXUr1+/3OuU5eTJk27H27dv73HOtGnT3I4HBQV5\nJROAyo0jOwCAEdghAQbzRoM+mvOhsmCHBAAwAgUJAGAEChJgs+3bt+vFF1/USy+9pP379zsdBzAG\nBQmwUV5enlauXKkPP/xQq1atUnp6utORAGNwU4MXlfXDJTExsdyv9/bbb7sd//Wvf+1xjl0PKu3a\ntavb8cOHD9uyfmV16NAhdevWTUFBQQoKCtK8efOcjgQYgx0SYKPLly/r3r17mjBhgoYPH65Dhw45\nHQkwBjskwGb5+flasWKFrl69qtGjR2vfvn3y8fGpsPW80UytKjeFI4M5GShIgI3q16+vjh07yt/f\nXy1atFCdOnWUm5vr9Sdp/K/HffRQVWgKRwZzMpRVzDiyA2zUs2dPHT58WCUlJcrLy1NBQYFCQkKc\njgUYgR0SYKPw8HDFxMRo6NChkqSEhAT5+vJ7ISBRkB6JZVlux1esWOFxzv37992O9+zZ0+McT3dg\neftOOk9/nqNHj3qc4+/v/lunU6dOHudU5PsklUl8fLzi4+OdjgEYh1/NAABGYIcEGIx+SKhO2CEB\nAIxAQQIAGIEjO8Bg9ENCdcIOCQBgBHZIj+Crr75yO759+3aPc2rUqOF2PCUlxeMcT7dWe1tWVpbb\n8e7du5f7te7du+fxWkBAQLlfD0D1wQ4JAGAEdkiAjY4cOaI33nhDP/jBDyRJbdq00Zw5cxxOBZiB\nggTYrGvXrvr973/vdAzAOBzZAQCMwA4JsNn58+c1YcIE3bx5U5MmTVKPHj0qdD36IZGhsmSgID2C\nDRs2lHvOxIkT3Y5XZB+c/5Wfn+/xWnR0dLlfb/78+W7H7WqhXlk99dRTmjRpkvr166dLly5p9OjR\n2rNnjwIDAytsTfohkcGkDPRDAgwRHh6u/v37y8fHRy1atFCDBg083nYPVDcUJMBG27dv19q1ayVJ\n2dnZysnJUXh4uMOpADNwZAfYKDIyUm+++abS09NVVFSkpKSkCj2uAyoTChJgo6CgIK1atcrpGICR\nOLIDABiBHRJgMBr0oTqhINnE0+3dZT2M9FFkZGS4HR8/frzHOadPn3Y73rJlS49zJk2a5Hbc15dN\nN4BHw08PAIAR2CEBBnvcBn0050Nlwg4JAGAEChIAwAgUJMAB9+7dU1RUVJkdg4HqhveQHkHjxo3L\nPSchIaFc4yZo166dx2tBQUE2Jql6/vjHPyo4ONjpGIBR2CEBNrtw4YLOnz+v3r17Ox0FMAo7JMBm\nixYt0pw5c7R169YKX8tbfWuqcg8eMpiTgYIE2Gjr1q167rnn1Lx5c1vW88ZTHqpCDx4ymJOhrGJG\nQQJstH//fl26dEn79+/XN998o8DAQDVq1Ejdu3d3OhrgOAoSYKPf/va3pR8vX75cTZs2pRgB/8FN\nDQAAI7BDegSvvfaa2/H79+97nJOYmOh2vKCgoNzrl/Wg1P3797sd/+qrrzzOqVmzpttxT5nhHZMn\nT3Y6AmAUdkgAACOwQwIMRj8kVCfskAAARqAgAQCMwJEdYLDH7Yck0RMJlQcF6REEBAS4HZ86darH\nORMnTnQ7bllWudcva05kZPl/+DRr1szteOfOncv9WgDwqDiyAwAYgYIEADACR3aAjQoLCzVjxgzl\n5OTo/v37mjhxovr06eN0LMAIFCTARvv27VP79u312muv6cqVK3r55ZcpSMB/UJAAG/Xv37/042vX\nrik8PNzBNIBZKEiAA+Lj4/XNN99o1apVFb6WN5qpVeWmcGQwJwMFySY1atTw2msNHDjQ47WjR4+6\nHW/UqJHHOenp6Y+dCeWzceNGffnll3rrrbe0fft2+fj4VNhaj/vooarQFI4M5mQoq5hxlx1go8zM\nTF27dk2S9Mwzz8jlcik3N9fhVIAZKEiAjY4dO6b3339fknTjxg0VFBQoJCTE4VSAGShIgI3i4+OV\nm5ur4cOHa9y4cUpMTJSvL38NAYn3kABb1axZU0uXLnU6BmAkfjUDABiBHVIllJGRUe45cXFxHq95\nergqnEeDPlQn7JAAAEagIAEAjMCRHWAwbzTok2jSh8qBHRIAwAgUJACAETiyA2y2ePFiffHFFyou\nLtb48ePVt29fpyMBRqAgGczTQ0+zsrI8zgkMDHQ7PmXKFK9kwuM5fPiwzp07p02bNikvL0+DBg2i\nIAH/QUECbPT888+rQ4cOkqR69eqpsLBQLpdLfn5+DicDnEdBAmzk5+en2rVrS5I2b96sF154wZZi\n9Lj9a6pyDx4ymJOBggQ44JNPPtHmzZtLn/xd0R63f43TT4sgQ9XJUFYxoyABNvv000+1atUqvffe\ne6pb1/nfdgFTUJAAG92+fVuLFy/Wn//8Zz3xxBNOxwGMQkFyWF5ensdrI0aMKPfrjR492u14ixYt\nyv1a8L5du3YpLy/vgbseFy1apCZNmjiYCjADBQmwUVxcXJlPXgeqM57UAAAwAjskwGD0Q0J1wg4J\nAGAEChIAwAgc2QEG80Y/JHohobKgIDns7t27Hq9lZ2eX+/WGDx/+OHEAwDEc2QEAjEBBAgAYgYIE\n2Ozs2bOKiorS+vXrnY4CGIWCBNiooKBA8+bNU7du3ZyOAhiHggTYKDAwUGvWrFFYWJjTUQDjcJcd\nYCN/f3/5+9v7184bzdSqclM4MpiTgYIEVHGP++ihqtAUjgzmZCirmHFkBwAwAgUJAGAEjuwAG2Vm\nZmrRokW6cuWK/P39lZaWpuXLl9M9FhAFCbBV+/btlZyc7HQMwEgUJMBg9ENCdUJBctjOnTvLPadu\nXc93qXTq1Olx4gCAY7ipAQBgBAoSAMAIHNkBBvNGgz6JJn2oHNghAQCMQEECABiBIzuH9e7du9xz\nNm3a5PFaWXfgwQwLFizQiRMn5OPjo1mzZqlDhw5ORwKMQEECbHT06FFdvHhRmzZt0oULFzRr1qwy\nf8EAqhOO7AAbHTp0SFFRUZKk1q1b6+bNm7pz547DqQAzsEMCbHTjxg21a9eu9PPQ0FBlZ2crKCio\nQtd93P41VbkHDxnMyUBBAhxkWZYt6zxu/xqnH19EhqqTgX5IgCHCwsJ048aN0s+vX7+uhg0bOpgI\nMAcFCbBRjx49lJaWJkk6deqUwsLCKvy4DqgsOLJzWNu2bT1ec7lcNiaBHTp16qR27dopPj5ePj4+\nmjt3rtORAGNQkACbvfnmm05HAIzEkR0AwAjskACD0aAP1Qk7JACAEShIAAAjUJAAAEagIAEAjEBB\nAgAYgYIEADACBQkAYAQfy67HDQMAUAZ2SAAAI1CQAABGoCABAIxAQQIAGIGCBAAwAgUJAGAEChIA\nwAgUJMAQCxYsUFxcnOLj43Xy5MkHrn322WcaPHiw4uLitHLlStvXP3z4sIYOHar4+HjNnDlTJSUl\ntmf4r6VLl2rUqFEVsv73Zbh27ZqGDRumwYMHKzEx0ZEMGzZsUFxcnIYNG6b58+dXWAZJOnv2rKKi\norR+/frvXKuQ70kLgOOOHDlijRs3zrIsyzp//rw1dOjQB67369fPunr1quVyuaxhw4ZZ586ds3X9\n6Oho69q1a5ZlWdbkyZOt/fv3e3X9h8lgWZZ17tw5Ky4uzho5cqTX13+YDL/85S+tPXv2WJZlWUlJ\nSdaVK1dszXD79m2rT58+VlFRkWVZljV27Fjr+PHjXs9gWZZ19+5da+TIkVZCQoKVnJz8nesV8T3J\nDgkwwKFDhxQVFSVJat26tW7evKk7d+5Iki5duqTg4GA1btxYvr6+ioiI0KFDh2xbX5JSUlLUqFEj\nSVJoaKjy8vK8uv7DZJCkhQsXaurUqV5f+2EylJSU6IsvvlBkZKQkae7cuWrSpImtGQICAhQQEKCC\nggIVFxersLBQwcHBXs8gSYGBgVqzZo3CwsK+c62ivicpSIABbty4oZCQkNLPQ0NDlZ2dLUnKzs5W\naGio22t2rC9JQUFBkqTr16/r4MGDioiI8Or6D5MhJSVFXbt2VdOmTb2+9sNkyM3NVZ06dfSb3/xG\nw4YN09KlS23PUKNGDb3++uuKiopSnz599OMf/1gtW7askBz+/v6qWbOm22sV9T1JQQIMZDn8iEl3\n6+fk5GjChAmaO3fuAz8w7ciQn5+vlJQUjR07tsLX9ZTBsixlZWVp9OjRWr9+vU6fPq39+/fbmuHO\nnTtavXq1UlNTlZ6erhMnTujMmTMVnsEuFCTAAGFhYbpx40bp59evX1fDhg3dXsvKynJ7jFJR60vf\n/iB87bXXNGXKFPXs2dOraz9MhsOHDys3N1cjRozQpEmTdOrUKS1YsMDWDCEhIWrSpIlatGghPz8/\ndevWTefOnbM1w4ULF9S8eXOFhoYqMDBQXbp0UWZmptczlDejt74nKUiAAXr06KG0tDRJ0qlTpxQW\nFlZ6TNasWTPduXNHly9fVnFxsfbt26cePXrYtr707Xs3Y8aM0QsvvODVdR82Q2xsrHbt2qWPPvpI\nK1asULt27TRr1ixbM/j7+6t58+b6+uuvS69XxHFZWRmaNm2qCxcu6N69e5KkzMxMPfXUU17P8H0q\n6nuS9hOAIZYsWaJjx47Jx8dHc+fO1enTp1W3bl1FR0fr888/15IlSyRJffv21SuvvGLb+j179tTz\nzz+vjh07ln7twIEDFRcXZ1uG6Ojo0q+5fPmyZs6cqeTkZK+v/30ZLl68qBkzZsiyLLVp00ZJSUny\n9fX+7/VlZdi4caNSUlLk5+enjh076u233/b6+tK3xW7RokW6cuWK/P39FR4ersjISDVr1qzCvicp\nSAAAI3BkBwAwAgUJAGAEChIAwAgUJACAEShIAAAjUJAAAEagIAEAjEBBAgAYgYIEADDC/wO+z+jh\n+PON2AAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<matplotlib.figure.Figure at 0x7f164bb7d630>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "metadata": {
        "id": "2PUZczqWdQLP",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "**Using 'nn.Sequential()**\n",
        "\n",
        "**from PyTorch to build the same network by passing tensors**\n",
        "\n",
        "**sequentially through the operations.**"
      ]
    },
    {
      "metadata": {
        "id": "IOaXQVPNg4kS",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 395
        },
        "outputId": "1eb0fe9f-81ef-48cf-a129-59b807221f00"
      },
      "cell_type": "code",
      "source": [
        "### set hyperparams \n",
        "\n",
        "input_size = 784\n",
        "hidden_sizes = [128, 64]\n",
        "output_size = 10 \n",
        "\n",
        "### build a feed-forward network\n",
        "\n",
        "model = nn.Sequential(nn.Linear(input_size, hidden_sizes[0]),\n",
        "                      nn.ReLU(),\n",
        "                      nn.Linear(hidden_sizes[0], hidden_sizes[1]),\n",
        "                      nn.ReLU(),\n",
        "                      nn.Linear(hidden_sizes[1], output_size),\n",
        "                      nn.Softmax(dim=1)) \n",
        "print(model)  \n",
        "print('')                      \n",
        "\n",
        "### forward pass through the network \n",
        "### and display the output \n",
        "\n",
        "images, labels = next(iter(trainloader))\n",
        "images.resize_(images.shape[0], 1, 784)\n",
        "ps = model.forward(images[0,:])\n",
        "view_classify(images[0].view(1, 28, 28), ps)                         "
      ],
      "execution_count": 154,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Sequential(\n",
            "  (0): Linear(in_features=784, out_features=128, bias=True)\n",
            "  (1): ReLU()\n",
            "  (2): Linear(in_features=128, out_features=64, bias=True)\n",
            "  (3): ReLU()\n",
            "  (4): Linear(in_features=64, out_features=10, bias=True)\n",
            "  (5): Softmax()\n",
            ")\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaQAAADhCAYAAACHrsFeAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAGYRJREFUeJzt3Xl0zWfix/FPFrFLEyRqbTGmLTWo\n9ow1pInE8uuvepBQy6GKaSlOaxBLoiYGhzEzmsGodtqooUNsHZKo0unYWstwYhnLOTW2BllsiU5y\nc39/zPSe49d7I7j3+32SvF9/yfPN9z6favjkefJ1Hz+n0+kUAAA287c7AAAAEoUEADAEhQQAMAKF\nBAAwAoUEADAChQQAMAKFBMAnnE6nPvzwQ/Xr108xMTGKiopSUlKSbt26JUmaNm2a/vCHP/g0w7Rp\n0/Tzn/9csbGxiomJUWxsrJYsWSKHw/FAr3Px4kU988wzDzz/sGHDtHnz5h+NZ2dnq1+/fpKkpUuX\nasaMGZKkESNG6Pjx45KkTz/99IHnK+8oJAA+sWjRIm3btk2rVq1SRkaGtmzZoqKiIo0dO1ZW/vPH\n4cOHKz09XRkZGVq/fr327t1r+1/24eHh+uyzz340/tFHH6l169ZyOBxauHChDcnsRSEB8Lr8/Hyl\npqZq/vz5Cg8PlyTVqFFDs2fP1ujRo39USEeOHNErr7yi2NhY9enTR3v37pUkFRcXa8aMGYqJiVF0\ndLTGjx+v27dvexy/n1q1aunll1/Wnj17JP1nBbNkyRL17t1bhw8fVn5+viZOnKiYmBj16dNHf/zj\nH++5/8MPP1Tv3r0VGRmpzz//XJJUUlKiOXPmKCYmRpGRkZoyZYqKiopc95w+fVoDBgxQRESEZs6c\nKYfD4XHFFRkZqYMHD2rkyJG6deuWYmNj9cknn2js2LGuzykpKVHnzp118uTJsvyvKFcoJABed/To\nUTVo0EAtWrS4Z7xq1aqKjIyUv/+9f/XMnj1br732mtLT0zVmzBglJiZKkv7+97/r4sWLSk9PV2Zm\nplq2bKkjR454HC+L4uJiBQUFuT7OysrSX//6V3Xo0EG/+c1vFBwcrIyMDK1Zs0Z//vOfdfDgQUmS\nw+GQw+HQ9u3bNXfuXM2aNUtFRUXasWOHDh48qM8++0zbt2/X8ePHtW3bNtfrHzhwQKmpqUpPT9c3\n33yjXbt23TfjvHnzFBAQoPT0dMXGxmr//v3Ky8uTJB0+fFh16tTR008/Xab/3vKEQgLgdfn5+apb\nt26ZP3/Tpk3q3bu3JOm5557ThQsXJEmhoaE6d+6cduzYocLCQk2aNEndunXzOH4/OTk52rBhg6Kj\no11jERERroL88ssvNWTIEEnSY489pujoaNdqSpL69+8vSerSpYuKi4v1r3/9SzExMdqwYYOqVKmi\nqlWr6tlnn3Xll6SYmBhVr15d1atXV0REhP7xj3+U+fdFkurWrauOHTsqIyNDkrRjxw716dPngV6j\nvKCQAHhdSEiIsrOzy/z5W7du1YABAxQTE6NRo0a5tvTatm2rmTNnKjU1VV26dNHbb7+tmzdvehx3\n5+OPP1ZsbKxiY2M1evRoDRw40FV+khQcHOz6dW5ururUqeP6uE6dOsrJybnnv+sHtWvX1s2bN5Wb\nm6upU6e6HprYuXPnPVuSoaGhP7rnQfXt29f1M6edO3dSSABQVu3atVNOTo7ribEfFBUVacmSJSos\nLHSNZWdna+bMmUpOTlZGRoZWrlx5zz2xsbFKTU3Vrl27VFhYqFWrVpU6/v/98FBDenq6Nm7cqGHD\nhnnMXa9ePeXn57s+zs/PV7169Vwf37hx455fBwcHa8mSJQoMDNTWrVuVnp6uiIiIe17T3T0PKjo6\nWllZWfryyy9VvXp1tWzZ8oFfozygkAB4XZ06dTR69GhNnTpV58+flyQVFhZq9uzZOnHihKpXr+76\n3NzcXNWoUUPNmzdXcXGx1q1bJ0m6c+eONmzYoJSUFEn/2UJr3ry5JHkcf1Q9evRwzZ+bm6sdO3ao\nR48erutbt26VJO3Zs0fVq1dX06ZNlZOTo1atWikoKEinTp3SkSNHVFBQ4LonMzNT33//vQoKCvTV\nV1+pY8eO981RpUoVlZSUuB7UqF27trp166Y5c+bcs7qraALtDgCgYpowYYKCg4P1i1/8Qg6HQ/7+\n/nrxxReVlJR0z+c99dRT6t69u2JiYlS3bl1NmzZNhw8f1rBhw/TBBx8oISFBvXr1UkBAgJo1a6b5\n8+dLksfxRzFp0iQlJSUpNjZW/v7+GjNmjNq2bauLFy+qRo0aKikpUb9+/XT37l0lJycrMDBQo0aN\n0tSpU5WWlqaOHTtq6tSpmjFjhtq2bStJ6ty5s4YPH67s7Gz16NFD3bp10+XLl0vNUb9+fT333HPq\n2bOnVqxYoQ4dOqhv377KzMyssNt1kuTHeUgAYL5jx47p3Xff1fr16+2O4jNs2QGA4YqLi5WSklLq\nz78qAgoJAAx24sQJRUdHKywsTC+99JLdcXyKLTsAgBFYIQEAjMBTdoDBiosdyssruP8n+lBISA0y\nkMFrGerXr+3xGiskwGCBgQF2RyADGSzLQCEBAIxAIQEAjEAhAQCMQCEBAIxAIQEAjEAhAQCMQCEB\nAIzAP4wFDPY/b29+5Nf4YFqkF5IAvscKCQBgBAoJAGAEtuwAC5WUlCgxMVFnzpxRlSpVlJSUpBYt\nWtgdCzACKyTAQjt37tStW7e0du1aJScna+HChXZHAoxBIQEW+vbbb9W2bVtJUtOmTXX58mU5HA6b\nUwFmYMsOsFCrVq300UcfacSIETp//rwuXLigvLw81atXz2dzlvZ2/1a+BhnIcD8UEmChiIgIHT58\nWK+++qp++tOfqnnz5vL1oc3Xrt16pPvr16/9yK/xqMhQcTKUVmYUEmCxyZMnu34dFRWlunXr2pgG\nMAc/QwIsdOrUKU2fPl2S9Le//U3PPPOM/P35YwhIrJAAS7Vq1UpOp1MDBgxQ1apVtWjRIrsjAcag\nkAAL+fv7a/78+XbHAIzEXgEAwAiskACDbV38v7Y/VQVYhRUSAMAIFBIAwAhs2QEG4zwkVCaskAAA\nRqCQAABGYMsOsNCdO3c0depU3bhxQ0VFRXrzzTfVrVs3u2MBRqCQAAtt3LhRTz75pN5++21lZ2dr\nxIgRSk9PtzsWYAS27AALhYSEKD8/X5J08+ZNhYSE2JwIMAcrJMBCffv2VVpamqKjo3Xz5k2tWLHC\n7kiAMSgkwEKbN29Ww4YNtWrVKp06dUoJCQlKS0vz6Zwc0EeG8pKBQgIsdPjwYXXt2lWS9NRTT+nq\n1atyOBwKCAjw2Zwc0EcGkzKUVmb8DAmwULNmzXT06FFJ0qVLl1SzZk2flhFQnrBCAiwUFxenhIQE\nDR06VMXFxUpKSrI7EmAMCgmwUM2aNfW73/3O7hiAkSikCmbzZvfvffbPf/7T4z0/HKntDfXr1/d4\n7eDBg27HGzdu7LX5AZRfFBJgMM5DQmXCQw0AACNQSAAAI7BlBxjMG+ch/YBzkWA6VkgAACOwQiqH\nVq5c6fHauHHj3I77+fl5vKe0aw/q+vXrHq95yp2YmOjxHn9/vmcCKgv+tAMAjMAKCbDQX/7yF23Z\nssX1cVZWlo4cOWJjIsAcFBJgoYEDB2rgwIGSpK+//lrbt2+3ORFgDrbsAJukpKTojTfesDsGYAxW\nSIANjh07pscff7zUt1rytkc5w6Yin8FDBnMyUEiADdavX6/+/ftbOufDvgVRRTiDhwzmZCitzCgk\nmzkcDo/XPD1CnZKS4qs4PpWcnOx2vE2bNh7v+eHnLRXNgQMHNHPmTLtjAEbhZ0iAxbKzs1WzZk0F\nBQXZHQUwCoUEWOzatWsKDQ21OwZgHAoJsFibNm30/vvv2x0DMA6FBAAwAg81AAbjgD5UJhSSF929\ne9fjtbVr17odP3TokMd7li1b9siZyoOwsDC7IwAwAFt2AAAjsEICDPYwB/RxEB/KK1ZIAAAjUEgA\nACNQSIDFtmzZopdeekmvvPKKdu/ebXccwBgUEmChvLw8paSkaM2aNVq+fLl27txpdyTAGDzU8BBO\nnjzpdrxPnz4e77lw4YKv4pRJaccc1KhRw+34+fPnfRXnHk2aNLFkHhPs27dPnTp1Uq1atVSrVi3N\nnTvX7kiAMVghARa6ePGi7t69q3HjxmnIkCHat2+f3ZEAY7BCAiyWn5+v9957T5cvX9bw4cO1a9cu\n+fn5ee31fXF4WkU+FI4M5mSgkAAL1a1bV+3bt1dgYKCaNm2qmjVrKjc3V3Xr1vXaHN5+q6GKcCgc\nGczJUFqZsWUHWKhr167av3+/SkpKlJeXp4KCAoWEhNgdCzACKyTAQuHh4YqJidGgQYMkSTNnzpS/\nP98XAhKFJKfT6XY8PT3d4z1LlixxO27Vk3SlPTG3YcMGt+OnTp3yeM+vfvWrR870KJYuXerxmqff\n6/IsPj5e8fHxdscAjMO3ZgAAI1T6FRJgMs5DQmXCCgkAYAQKCQBgBLbsAIM9zHlInnBOEkzHCgkA\nYIRKv0JyOBxux9esWePxni+++MJXccrkxRdf9HgtJSXF7fi6det8FQcAvIIVEgDACJV+hQRY6cCB\nA5o4caJ+8pOfSJJatWqlWbNm2ZwKMAOFBFjshRde0O9//3u7YwDGYcsOAGAEVkiAxc6ePatx48bp\nxo0bGj9+vLp06WLJvI9yhk1FPoOHDOZkqPSFVFxc7HZ806ZNFicpu7Vr13q85unNYr15ABwe3hNP\nPKHx48erd+/eunDhgoYPH67MzEwFBQX5fO6HfQuiinAGDxnMycB5SIAhwsPD1adPH/n5+alp06aq\nV6+esrOz7Y4FGIFCAiy0ZcsWrVq1SpJ07do15eTkKDw83OZUgBkq/ZYdYKXIyEi988472rlzp4qK\nipSUlGTJdh1QHlBIgIVq1aql5cuX2x0DMBJbdgAAI7BCAgzGAX2oTCpFIV2+fNnjtdGjR7sdLyws\n9FWcSqtx48Zux6dMmWJxEgAmYssOAGCESrFCAsqrRzmgjwP5UN6wQgIAGIFCAgAYgUICbHD37l1F\nRUUpLS3N7iiAMSrFz5AKCgo8XsvMzLQwSeX27rvvuh1v2LChxUnst2zZMgUHB9sdAzAKKyTAYufO\nndPZs2fVo0cPu6MARqkUKyTAJAsWLNCsWbN8fsSJN8+sqchn8JDBnAwUEmChTZs2qV27dmrSpInP\n5/LWOzxUhDN4yGBOhtLKjEICLLR7925duHBBu3fv1nfffaegoCA1aNBAnTt3tjsaYDsKCbDQb3/7\nW9evly5dqkaNGlFGwH/xUAMAwAiVYoXUsmVLj9c2btzodvzll1/2VRyfKikpcTvu72/N9x6HDh3y\neK1du3aWZCgvJkyYYHcEwCiskAAARqgUKySgvOI8JFQmrJAAAEagkAAARmDLDjDYw56HxFlIKI8q\nfSH17dvX7filS5ce+LWWLVvm8dqxY8ce+PU8/Wv+hIQEj/ds2LDB7fjEiRMfeP6H8fHHH3u8xlN2\nAErDlh0AwAgUEgDACJV+yw6wUmFhoaZNm6acnBx9//33euONN9SzZ0+7YwFGoJAAC+3atUtt2rTR\n66+/rkuXLmnUqFEUEvBfFBJgoT59+rh+feXKFYWHh9uYBjALhQTYID4+Xt99952WL1/uk9f39gFq\nFflQODKYk6HSF5KnNx1t0KDBA7/WnDlzHjVOmRQVFXm8Fhoa6rV5qlev7vFaVFSU23Grfg/Ku7Vr\n1+rkyZOaMmWKtmzZIj8/P6++vjffbqgiHApHBnMylFZmPGUHWCgrK0tXrlyRJD399NNyOBzKzc21\nORVgBgoJsNDBgwf1wQcfSJKuX7+ugoIChYSE2JwKMAOFBFgoPj5eubm5GjJkiMaMGaPZs2dbdlYV\nYLpK/zMkwErVqlXT4sWL7Y4BGIlvzQAARmCFZLA7d+64HR84cKDHezIyMtyOP8xTXMnJyR6vvfXW\nWw/8enhwHNCHyoQVEgDACBQSAMAIbNkBBnvYA/r+Pw7sQ3nACgkAYAQKCQBgBLbsAIstXLhQhw4d\nUnFxscaOHatevXrZHQkwAoVksPHjx7sdz8zM9Oo8QUFBbsefffZZr84Daf/+/Tpz5ozWrVunvLw8\n9e/fn0IC/otCAiz0/PPPq23btpKkOnXqqLCwUA6HQwEBATYnA+xHIQEWCggIUI0aNSRJ69evV/fu\n3S0po0c9v6Yin8FDBnMyUEiADT7//HOtX7/e9c7fvvao59fY/W4RZKg4GUorMwoJsNhXX32l5cuX\n6/3331ft2vZ/twuYgkICLHTr1i0tXLhQf/rTn/TYY4/ZHQcwCoVkswMHDni8tn37dksyLFiwwO14\nz549LZm/Mtm2bZvy8vI0adIk19iCBQvUsGFDG1MBZqCQAAvFxcUpLi7O7hiAkXinBgCAEVghAQbj\nPCRUJqyQAABGoJAAAEZgyw4wmDfOQ+IsJJQXFJLN+vbt6/Fafn6+JRnat29vyTwAUBq27AAARqCQ\nAABGoJAAi50+fVpRUVFavXq13VEAo1BIgIUKCgo0d+5cderUye4ogHEoJMBCQUFBWrlypcLCwuyO\nAhiHp+y86N///rfHa5MnT3Y7fuPGDa9m8HTs+JgxYzzew3fr1gkMDFRgoLV/7LxxmFpFPhSODOZk\noJCACu5R33qoIhwKRwZzMpRWZmzZAQCMQCEBAIzAlh1goaysLC1YsECXLl1SYGCgMjIytHTpUk6P\nBUQhAZZq06aNUlNT7Y4BGIlCAgzGeUioTCgkLyosLPR4bcWKFV6bp1mzZh6vffHFF27HQ0NDvTY/\nAPgCDzUAAIxAIQEAjMCWHWCwRz2gj8P5UJ6wQgIAGIFCAgAYgS07L6pWrZrHa/Hx8W7H165d6/Ge\n+fPnux0fOnSox3t4ms588+bN09GjR+Xn56eEhAS1bdvW7kiAESgkwEJff/21zp8/r3Xr1uncuXNK\nSEjQunXr7I4FGIEtO8BC+/btU1RUlCSpRYsWunHjhm7fvm1zKsAMrJAAC12/fl2tW7d2fRwaGqpr\n166pVq1aPpnPW+fWVOQzeMhgTgYKCbCR0+n06et7422HKsIZPGQwJwPnIQGGCAsL0/Xr110fX716\nVfXr17cxEWAOCgmwUJcuXZSRkSFJOn78uMLCwny2XQeUN2zZeVHVqlU9Xlu9evUDjaNi6tChg1q3\nbq34+Hj5+fkpMTHR7kiAMSgkwGLvvPOO3REAI7FlBwAwAiskwGAc0IfKhBUSAMAIFBIAwAgUEgDA\nCBQSAMAIFBIAwAgUEgDACBQSAMAIfk5fv90wAABlwAoJAGAECgkAYAQKCQBgBAoJAGAECgkAYAQK\nCQBgBAoJAGAECgkwxLx58xQXF6f4+HgdO3bsnmt79+7VgAEDFBcXp5SUFMvn379/vwYNGqT4+HhN\nnz5dJSUllmf4weLFizVs2DCfzH+/DFeuXNHgwYM1YMAAzZ4925YMn3zyieLi4jR48GAlJyf7LIMk\nnT59WlFRUVq9evWPrvnka9IJwHYHDhxwjhkzxul0Op1nz551Dho06J7rvXv3dl6+fNnpcDicgwcP\ndp45c8bS+aOjo51XrlxxOp1O54QJE5y7d+/26vxlyeB0Op1nzpxxxsXFOYcOHer1+cuS4a233nJm\nZmY6nU6nMykpyXnp0iVLM9y6dcvZs2dPZ1FRkdPpdDpHjhzpPHLkiNczOJ1O5507d5xDhw51zpw5\n05mamvqj6774mmSFBBhg3759ioqKkiS1aNFCN27c0O3btyVJFy5cUHBwsB5//HH5+/srIiJC+/bt\ns2x+SUpLS1ODBg0kSaGhocrLy/Pq/GXJIEnz58/X5MmTvT53WTKUlJTo0KFDioyMlCQlJiaqYcOG\nlmaoUqWKqlSpooKCAhUXF6uwsFDBwcFezyBJQUFBWrlypcLCwn50zVdfkxQSYIDr168rJCTE9XFo\naKiuXbsmSbp27ZpCQ0PdXrNifkmqVauWJOnq1avas2ePIiIivDp/WTKkpaXphRdeUKNGjbw+d1ky\n5ObmqmbNmvr1r3+twYMHa/HixZZnqFq1qt58801FRUWpZ8+e+tnPfqYnn3zSJzkCAwNVrVo1t9d8\n9TVJIQEGctr8FpPu5s/JydG4ceOUmJh4z1+YVmTIz89XWlqaRo4c6fN5PWVwOp3Kzs7W8OHDtXr1\nap04cUK7d++2NMPt27e1YsUKpaena+fOnTp69KhOnTrl8wxWoZAAA4SFhen69euuj69evar69eu7\nvZadne12G8VX80v/+Yvw9ddf16RJk9S1a1evzl2WDPv371dubq5effVVjR8/XsePH9e8efMszRAS\nEqKGDRuqadOmCggIUKdOnXTmzBlLM5w7d05NmjRRaGiogoKC1LFjR2VlZXk9w4Nm9NbXJIUEGKBL\nly7KyMiQJB0/flxhYWGubbLGjRvr9u3bunjxooqLi7Vr1y516dLFsvml//zsZsSIEerevbtX5y1r\nhtjYWG3btk2ffvqp3nvvPbVu3VoJCQmWZggMDFSTJk307bffuq77YrustAyNGjXSuXPndPfuXUlS\nVlaWnnjiCa9nuB9ffU1y/ARgiEWLFungwYPy8/NTYmKiTpw4odq1ays6OlrffPONFi1aJEnq1auX\nXnvtNcvm79q1q55//nm1b9/e9bn9+vVTXFycZRmio6Ndn3Px4kVNnz5dqampXp//fhnOnz+vadOm\nyel0qlWrVkpKSpK/v/e/ry8tw9q1a5WWlqaAgAC1b99ev/zlL70+v/SfsluwYIEuXbqkwMBAhYeH\nKzIyUo0bN/bZ1ySFBAAwAlt2AAAjUEgAACNQSAAAI1BIAAAjUEgAACNQSAAAI1BIAAAjUEgAACNQ\nSAAAI/wfUUHu6pSJTV4AAAAASUVORK5CYII=\n",
            "text/plain": [
              "<matplotlib.figure.Figure at 0x7f164b66b278>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "metadata": {
        "id": "wjMvCx9wmnut",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "outputId": "31ffab04-5a80-4937-9ed8-5640b718f65a"
      },
      "cell_type": "code",
      "source": [
        "### print the weights in the input layer\n",
        "\n",
        "print(model[0])\n",
        "print('')\n",
        "model[0].weight"
      ],
      "execution_count": 155,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Linear(in_features=784, out_features=128, bias=True)\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Parameter containing:\n",
              "tensor([[ 0.0022, -0.0037,  0.0093,  ..., -0.0269, -0.0047, -0.0287],\n",
              "        [-0.0180,  0.0298,  0.0276,  ...,  0.0118,  0.0047, -0.0055],\n",
              "        [ 0.0101,  0.0275,  0.0312,  ...,  0.0128, -0.0204, -0.0125],\n",
              "        ...,\n",
              "        [-0.0168,  0.0205,  0.0132,  ..., -0.0006, -0.0076,  0.0316],\n",
              "        [ 0.0066, -0.0048,  0.0202,  ..., -0.0207, -0.0336, -0.0101],\n",
              "        [-0.0001, -0.0269, -0.0064,  ..., -0.0257, -0.0328,  0.0205]],\n",
              "       requires_grad=True)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 155
        }
      ]
    },
    {
      "metadata": {
        "id": "6VahUGDHnZlr",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 153
        },
        "outputId": "14e751bd-6921-47d2-8b11-da3a679c8e8e"
      },
      "cell_type": "code",
      "source": [
        "### name the layers and operations in key-value pairs\n",
        "\n",
        "model = nn.Sequential(OrderedDict([\n",
        "                                 ('input_layer', nn.Linear(input_size, hidden_sizes[0])),\n",
        "                                 ('relu1', nn.ReLU()),\n",
        "                                 ('hidden_layer', nn.Linear(hidden_sizes[0], hidden_sizes[1])),\n",
        "                                 ('relu2', nn.ReLU()),\n",
        "                                 ('output_layer', nn.Linear(hidden_sizes[1], output_size)),\n",
        "                                 ('softmax', nn.Softmax(dim=1))]))  \n",
        "print(model)\n"
      ],
      "execution_count": 156,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Sequential(\n",
            "  (input_layer): Linear(in_features=784, out_features=128, bias=True)\n",
            "  (relu1): ReLU()\n",
            "  (hidden_layer): Linear(in_features=128, out_features=64, bias=True)\n",
            "  (relu2): ReLU()\n",
            "  (output_layer): Linear(in_features=64, out_features=10, bias=True)\n",
            "  (softmax): Softmax()\n",
            ")\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "Yot0X5yDrT4S",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "**Now I can access the layers and functions**\n",
        "\n",
        "**either by index position or by key name.**"
      ]
    },
    {
      "metadata": {
        "id": "28JiGSd7ry4w",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "bcc739ff-adc1-4759-e8aa-fb8e58411ba3"
      },
      "cell_type": "code",
      "source": [
        "### use index position and key name \n",
        "### to access the input layer\n",
        "\n",
        "print(model[0])\n",
        "print(model.input_layer)\n",
        "\n"
      ],
      "execution_count": 157,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Linear(in_features=784, out_features=128, bias=True)\n",
            "Linear(in_features=784, out_features=128, bias=True)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "kX-j-NnHBefx",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 2346
        },
        "outputId": "5953f91d-c4c3-4d5d-8e33-72beda5873ce"
      },
      "cell_type": "code",
      "source": [
        "!pip freeze"
      ],
      "execution_count": 158,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "absl-py==0.6.1\n",
            "altair==2.2.2\n",
            "astor==0.7.1\n",
            "atomicwrites==1.2.1\n",
            "attrs==18.2.0\n",
            "beautifulsoup4==4.6.3\n",
            "bleach==3.0.2\n",
            "cachetools==3.0.0\n",
            "certifi==2018.10.15\n",
            "chardet==3.0.4\n",
            "crcmod==1.7\n",
            "cycler==0.10.0\n",
            "cymem==2.0.2\n",
            "cytoolz==0.9.0.1\n",
            "decorator==4.3.0\n",
            "defusedxml==0.5.0\n",
            "dill==0.2.8.2\n",
            "entrypoints==0.2.3\n",
            "future==0.16.0\n",
            "gast==0.2.0\n",
            "google-api-core==1.5.2\n",
            "google-api-python-client==1.6.7\n",
            "google-auth==1.4.2\n",
            "google-auth-httplib2==0.0.3\n",
            "google-auth-oauthlib==0.2.0\n",
            "google-cloud-bigquery==1.1.0\n",
            "google-cloud-core==0.28.1\n",
            "google-cloud-language==1.0.2\n",
            "google-cloud-storage==1.8.0\n",
            "google-cloud-translate==1.3.1\n",
            "google-colab==0.0.1a1\n",
            "google-resumable-media==0.3.1\n",
            "googleapis-common-protos==1.5.5\n",
            "grpcio==1.15.0\n",
            "h5py==2.8.0\n",
            "helper==2.4.2\n",
            "httplib2==0.11.3\n",
            "idna==2.6\n",
            "ipykernel==4.6.1\n",
            "ipython==5.5.0\n",
            "ipython-genutils==0.2.0\n",
            "Jinja2==2.10\n",
            "joblib==0.13.0\n",
            "jsonschema==2.6.0\n",
            "jupyter-client==5.2.3\n",
            "jupyter-core==4.4.0\n",
            "Keras==2.2.4\n",
            "Keras-Applications==1.0.6\n",
            "Keras-Preprocessing==1.0.5\n",
            "Markdown==3.0.1\n",
            "MarkupSafe==1.1.0\n",
            "matplotlib==2.1.2\n",
            "mistune==0.8.4\n",
            "more-itertools==4.3.0\n",
            "mpmath==1.0.0\n",
            "msgpack==0.5.6\n",
            "msgpack-numpy==0.4.3.2\n",
            "murmurhash==1.0.1\n",
            "nbconvert==5.4.0\n",
            "nbformat==4.4.0\n",
            "networkx==2.2\n",
            "nltk==3.2.5\n",
            "notebook==5.2.2\n",
            "numpy==1.14.6\n",
            "oauth2client==4.1.3\n",
            "oauthlib==2.1.0\n",
            "olefile==0.46\n",
            "opencv-python==3.4.3.18\n",
            "pandas==0.22.0\n",
            "pandas-gbq==0.4.1\n",
            "pandocfilters==1.4.2\n",
            "patsy==0.5.1\n",
            "pexpect==4.6.0\n",
            "pickleshare==0.7.5\n",
            "Pillow==5.3.0\n",
            "plac==0.9.6\n",
            "plotly==1.12.12\n",
            "pluggy==0.8.0\n",
            "portpicker==1.2.0\n",
            "preshed==2.0.1\n",
            "prompt-toolkit==1.0.15\n",
            "protobuf==3.6.1\n",
            "psutil==5.4.8\n",
            "ptyprocess==0.6.0\n",
            "py==1.7.0\n",
            "pyasn1==0.4.4\n",
            "pyasn1-modules==0.2.2\n",
            "Pygments==2.1.3\n",
            "pygobject==3.26.1\n",
            "pymc3==3.5\n",
            "pyparsing==2.3.0\n",
            "pystache==0.5.4\n",
            "pytest==3.10.1\n",
            "python-apt==1.6.3\n",
            "python-dateutil==2.5.3\n",
            "pytz==2018.7\n",
            "PyWavelets==1.0.1\n",
            "PyYAML==3.13\n",
            "pyzmq==17.0.0\n",
            "regex==2018.1.10\n",
            "requests==2.18.4\n",
            "requests-oauthlib==1.0.0\n",
            "rsa==4.0\n",
            "scikit-image==0.13.1\n",
            "scikit-learn==0.19.2\n",
            "scipy==1.1.0\n",
            "seaborn==0.7.1\n",
            "simplegeneric==0.8.1\n",
            "six==1.11.0\n",
            "spacy==2.0.16\n",
            "statsmodels==0.8.0\n",
            "sympy==1.1.1\n",
            "tensorboard==1.12.0\n",
            "tensorflow==1.12.0\n",
            "tensorflow-hub==0.1.1\n",
            "tensorflow-probability==0.5.0\n",
            "termcolor==1.1.0\n",
            "terminado==0.8.1\n",
            "testpath==0.4.2\n",
            "Theano==1.0.3\n",
            "thinc==6.12.0\n",
            "toolz==0.9.0\n",
            "torch==0.4.1\n",
            "torchvision==0.2.1\n",
            "tornado==4.5.3\n",
            "tqdm==4.28.1\n",
            "traitlets==4.3.2\n",
            "typing==3.6.6\n",
            "ujson==1.35\n",
            "uritemplate==3.0.0\n",
            "urllib3==1.22\n",
            "vega-datasets==0.5.0\n",
            "wcwidth==0.1.7\n",
            "webencodings==0.5.1\n",
            "Werkzeug==0.14.1\n",
            "wrapt==1.10.11\n",
            "xgboost==0.7.post4\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}