assignment9-3.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Untitled9.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyMt9+uHu2gv0A7q9Ou5O0OS",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/dlguswn3659/class-MachineLearning/blob/master/assignment9-3.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-sOoEaBxtNXA",
        "colab_type": "code",
        "outputId": "398e470c-d5fa-422f-ec02-bbd475c4bb24",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 338
        }
      },
      "source": [
        "%matplotlib inline\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "import matplotlib as mpl\n",
        "import seaborn as sns\n",
        "import matplotlib.animation as animation\n",
        "import operator\n",
        "from scipy import stats \n",
        "from mpl_toolkits.mplot3d import Axes3D\n",
        "from sklearn.datasets.samples_generator import make_regression \n",
        "import csv\n",
        "import math\n",
        "# from google.colab import files\n",
        "\n",
        "# uploaded = files.upload()\n",
        "\n",
        "file_data   = \"mnist.csv\"\n",
        "handle_file = open(file_data, \"r\")\n",
        "data        = handle_file.readlines()\n",
        "handle_file.close()\n",
        "\n",
        "size_row    = 28    # height of the image\n",
        "size_col    = 28    # width of the image\n",
        "\n",
        "num_image   = len(data)\n",
        "count       = 0     # count for the number of images\n",
        "\n",
        "#\n",
        "# normalize the values of the input data to be [0, 1]\n",
        "#\n",
        "def normalize(data):\n",
        "\n",
        "    data_normalized = (data - min(data)) / (max(data) - min(data))\n",
        "\n",
        "    return(data_normalized)\n",
        "\n",
        "#\n",
        "# example of distance function between two vectors x and y\n",
        "#\n",
        "def distance(x, y):\n",
        "\n",
        "    d = (x - y) ** 2\n",
        "    s = np.sum(d)\n",
        "    # r = np.sqrt(s)\n",
        "\n",
        "    return(s)\n",
        "\n",
        "#\n",
        "# make a matrix each column of which represents an images in a vector form\n",
        "#\n",
        "list_image  = np.empty((size_row * size_col, num_image), dtype=float)\n",
        "list_label  = np.empty(num_image, dtype=int)\n",
        "\n",
        "for line in data:\n",
        "\n",
        "    line_data   = line.split(',')\n",
        "    label       = line_data[0]\n",
        "    im_vector   = np.asfarray(line_data[1:])\n",
        "    im_vector   = normalize(im_vector)\n",
        "\n",
        "    list_label[count]       = label\n",
        "    list_image[:, count]    = im_vector\n",
        "\n",
        "    count += 1\n",
        "\n",
        "#\n",
        "# plot first 150 images out of 10,000 with their labels\n",
        "#\n",
        "f1 = plt.figure(1)\n",
        "\n",
        "for i in range(150):\n",
        "\n",
        "    label       = list_label[i]\n",
        "    im_vector   = list_image[:, i]\n",
        "    im_matrix   = im_vector.reshape((size_row, size_col))\n",
        "\n",
        "    plt.subplot(10, 15, i+1)\n",
        "    plt.title(label)\n",
        "    plt.imshow(im_matrix, cmap='Greys', interpolation='None')\n",
        "\n",
        "    frame   = plt.gca()\n",
        "    frame.axes.get_xaxis().set_visible(False)\n",
        "    frame.axes.get_yaxis().set_visible(False)\n",
        "\n",
        "\n",
        "#plt.show()\n",
        "\n",
        "#\n",
        "# plot the average image of all the images for each digit\n",
        "#\n",
        "f2 = plt.figure(2)\n",
        "\n",
        "im_average  = np.zeros((size_row * size_col, 10), dtype=float)\n",
        "im_count    = np.zeros(10, dtype=int)\n",
        "\n",
        "for i in range(num_image):\n",
        "\n",
        "    im_average[:, list_label[i]] += list_image[:, i]\n",
        "    im_count[list_label[i]] += 1\n",
        "\n",
        "for i in range(10):\n",
        "\n",
        "    im_average[:, i] /= im_count[i]\n",
        "\n",
        "    plt.subplot(2, 5, i+1)\n",
        "    plt.title(i)\n",
        "    plt.imshow(im_average[:,i].reshape((size_row, size_col)), cmap='Greys', interpolation='None')\n",
        "\n",
        "    frame   = plt.gca()\n",
        "    frame.axes.get_xaxis().set_visible(False)\n",
        "    frame.axes.get_yaxis().set_visible(False)\n",
        "\n",
        "plt.show()\n",
        "\n"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/statsmodels/tools/_testing.py:19: FutureWarning: pandas.util.testing is deprecated. Use the functions in the public API at pandas.testing instead.\n",
            "  import pandas.util.testing as tm\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/utils/deprecation.py:144: FutureWarning: The sklearn.datasets.samples_generator module is  deprecated in version 0.22 and will be removed in version 0.24. The corresponding classes / functions should instead be imported from sklearn.datasets. Anything that cannot be imported from sklearn.datasets is now part of the private API.\n",
            "  warnings.warn(message, FutureWarning)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "error",
          "ename": "FileNotFoundError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-2-3b19457ef0f0>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     16\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0mfile_data\u001b[0m   \u001b[0;34m=\u001b[0m \u001b[0;34m\"mnist.csv\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 18\u001b[0;31m \u001b[0mhandle_file\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfile_data\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"r\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     19\u001b[0m \u001b[0mdata\u001b[0m        \u001b[0;34m=\u001b[0m \u001b[0mhandle_file\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreadlines\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     20\u001b[0m \u001b[0mhandle_file\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'mnist.csv'"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "b_YcDHFWIRFP",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "theta = np.random.normal(size=(size_row*size_col))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DemnhlAQP0XJ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def sigmoid(z): \n",
        "  g = 1 / (1 + np.exp(-z)) \n",
        "  \n",
        "  return g "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DRugdPmoP2G5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "z_list = []\n",
        "for i in range(0, 10):\n",
        "  z_list.append(0.0)\n",
        "\n",
        "print(z_list)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xtbUZT3IP28F",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "for i in range(0, num_image):\n",
        "  label       = list_label[i]\n",
        "  im_vector   = list_image[:, i]\n",
        "  im_matrix   = im_vector.reshape((size_row, size_col))\n",
        "  y = 0\n",
        "  for ny in range(0, size_row):\n",
        "      for nz in range(0, size_col):\n",
        "        y = y + theta[size_col*ny + nz]*im_matrix[ny][nz]\n",
        "  # print(y)\n",
        "  z_list[label] = z_list[label] + sigmoid(y)\n",
        "  # print(sigmoid(y))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OWQYa_g6P4H2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#평균내기\n",
        "\n",
        "# for i in range(0, 10):\n",
        "  # z_list[i] = z_list[i] / label_list[i]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WUWOjwQQTDdR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#first hidden layer로 가기 위한 초기 fully connected u 생성\n",
        "w_tmp = np.ones(shape=(6000, 197, 785))\n",
        "v_tmp = np.ones(shape=(6000, 50, 197))\n",
        "u_tmp = np.ones(shape=(6000, 10, 50))\n",
        "\n",
        "matrix_u = np.zeros((197, 785))  #197줄\n",
        "matrix_u[0][0] = 1               #y레이어의 bias를 위한 값 1\n",
        "\n",
        "for i in range(1, 197):\n",
        "  for j in range(0, 785):\n",
        "    matrix_u[i][j] = np.random.normal()\n",
        "\n",
        "\n",
        "# matrix_u[196][0] = 0            #줄, 칸\n",
        "##############################################\n",
        "matrix_v = np.zeros((50, 197))  #50줄\n",
        "matrix_v[0][0] = 1               #y레이어의 bias를 위한 값 1\n",
        "\n",
        "for i in range(1, 50):\n",
        "  for j in range(0, 197):\n",
        "    matrix_v[i][j] = np.random.normal()\n",
        "\n",
        "\n",
        "# matrix_v[196][0] = 0            #줄, 칸\n",
        "##############################################\n",
        "matrix_w = np.zeros((10, 50))  #10줄\n",
        "# matrix_w[0][0] = 1               #y레이어의 bias를 위한 값 1\n",
        "\n",
        "for i in range(0, 10):\n",
        "  for j in range(0, 50):\n",
        "    matrix_w[i][j] = np.random.normal()\n",
        "\n",
        "w_tmp = matrix_w\n",
        "v_tmp = matrix_v\n",
        "u_tmp = matrix_u\n",
        "\n",
        "# matrix_w[196][0] = 0            #줄, 칸\n",
        "\n",
        "# print(matrix)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "idHIKSRGY0p9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import operator\n",
        "\n",
        "y_ = np.ones((197, 1))\n",
        "\n",
        "x_list = np.ones(shape=(6000, 785, 1))\n",
        "\n",
        "y__list = np.ones(shape=(6000, 197, 1))\n",
        "z__list = np.ones(shape=(6000, 50, 1))\n",
        "h__list = np.ones(shape=(6000, 10, 1))\n",
        "\n",
        "y_list = np.ones(shape=(6000, 197, 1))\n",
        "z_list = np.ones(shape=(6000, 50, 1))\n",
        "h_list = np.ones(shape=(6000, 10, 1))\n",
        "l_list = np.zeros(shape=(6000, 10, 1))\n",
        "\n",
        "y_list_tmp = np.ones(shape=(6000, 197, 1))\n",
        "z_list_tmp = np.ones(shape=(6000, 50, 1))\n",
        "h_list_tmp = np.ones(shape=(6000, 10, 1))\n",
        "y__list_tmp = np.ones(shape=(6000, 197, 1))\n",
        "z__list_tmp = np.ones(shape=(6000, 50, 1))\n",
        "h__list_tmp = np.ones(shape=(6000, 10, 1))\n",
        "\n",
        "# w_tmp = np.ones(shape=(6000, 197, 785))\n",
        "# v_tmp = np.ones(shape=(6000, 50, 197))\n",
        "# u_tmp = np.ones(shape=(6000, 10, 50))\n",
        "\n",
        "# matrix_u = np.zeros((197, 785))  #197줄\n",
        "# matrix_u[0][0] = 1               #y레이어의 bias를 위한 값 1\n",
        "\n",
        "\n",
        "#training set 6000개에 대해서\n",
        "for i in range(0, 6000):\n",
        "  # print(i)\n",
        "  label       = list_label[i]\n",
        "  bias   = [1]       #bias 1\n",
        "  im_vector = np.concatenate((bias, list_image[:, i]), axis = None)\n",
        "\n",
        "  # print(type(list_image[: , i]))\n",
        "  # print(list_image[:, i])\n",
        "  # print(im_vector)\n",
        "  im_matrix   = im_vector.reshape((size_row*size_col + 1, 1))\n",
        "  \n",
        "  x_list[i] = im_matrix\n",
        "\n",
        "  y_ = np.dot(matrix_u, im_matrix)  #x와 가중치 u를 곱해서 y_를 만듦\n",
        "  y__list[i] = y_\n",
        "  \n",
        "  y_vector = np.ones((197,1)) #bias\n",
        "  for j in range (1, 197):\n",
        "    y_vector[j][0] = 1 / (1 + math.exp(-y_[j][0]))\n",
        "    # sigmoid(y_[j][0])  #sigmoid 함수를 적용해서 y_에서 y를 만듦\n",
        "\n",
        "  y   = y_vector.reshape((197, 1))\n",
        "  y_list[i] = y\n",
        "\n",
        "  z_  = np.dot(matrix_v, y)  #y와 가중치 v를 곱해서 z_를 만듦\n",
        "  z__list[i] = z_\n",
        "  z_vector = np.ones((50,1)) #bias\n",
        "  for j in range (1, 50):\n",
        "    z_vector[j][0] = 1 / (1 + math.exp(-z_[j][0]))  #sigmoid 함수를 적용해서 y_에서 y를 만듦\n",
        "\n",
        "  z   = z_vector.reshape((50, 1))\n",
        "  z_list[i] = z\n",
        "  \n",
        "  # print(z)\n",
        "\n",
        "  h_  = np.dot(matrix_w, z)  #z와 가중치 w를 곱해서 h_를 만듦\n",
        "  h__list[i] = h_\n",
        "  \n",
        "  h_vector = np.ones((10,1)) #bias\n",
        "  for j in range (0, 10):\n",
        "    h_vector[j][0] = 1 / (1 + math.exp(-h_[j][0]))  #sigmoid 함수를 적용해서 h_에서 h를 만듦\n",
        "\n",
        "  h   = h_vector.reshape((10, 1))\n",
        "  # tmp = h.ravel()\n",
        "  # max_value = max(tmp)\n",
        "  # print(max_value)\n",
        "  # max_index = tmp.index(max_value)\n",
        "  index, value = max(enumerate(h), key=operator.itemgetter(1))\n",
        "  h_list[i] = h\n",
        "  l_list[i][index] = 1\n",
        "  # print(l_list[i])\n",
        "  # print(h)\n",
        "\n",
        "  y_list_tmp[i] = y_list[i]\n",
        "  z_list_tmp[i] = z_list[i]\n",
        "  h_list_tmp[i] = h_list[i]\n",
        "  y__list_tmp[i] = y__list[i]\n",
        "  z__list_tmp[i] = z__list[i]\n",
        "  h__list_tmp[i] = h__list[i]\n",
        "\n",
        "# print(l_list)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "i1AbB1lMv1yY",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def objective_function(size, label_List, h_List):\n",
        "  result = 0.0\n",
        "  for i in range(0, size):\n",
        "    for k in range(0, 10):\n",
        "      result = result + ((-1)*label_List[i][k]*math.log(h_List[i][k][0]) - (1-label_List[i][k])*math.log(1-h_List[i][k][0]))\n",
        "  result = result / size\n",
        "\n",
        "  return result"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "z7uMF29V6ZAh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def w_gredient_descent(learning_rate, k_value, n_value, size, label_List):\n",
        "  result = 0.0\n",
        "  for i in range(0, size):\n",
        "    result = result + (z_list_tmp[i][n_value][0]*(sigmoid(h_list_tmp[i][k_value][0]) - label_List[i][k_value][0]))\n",
        "  result = result / size\n",
        "  result = result * learning_rate\n",
        "  final = w_tmp[k_value][n_value] - result\n",
        "\n",
        "  return final"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FQMnniqxLxzl",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def v_gredient_descent(learning_rate, k_value, n_value, size, label_List):\n",
        "  result = 0.0\n",
        "  for i in range(0, size):\n",
        "    result = result + (y_list_tmp[i][n_value][0]*w_tmp[0][k_value]*(sigmoid(h_list_tmp[i][0][0]) - label_List[i][0][0])*(sigmoid(z__list_tmp[i][k_value][0])*(1 - sigmoid(z__list_tmp[i][k_value][0]))))\n",
        "  result = result / size\n",
        "  result = result * learning_rate\n",
        "  final = v_tmp[k_value][n_value] - result\n",
        "\n",
        "  return final"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UabSYhhIO6JU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def u_gredient_descent(learning_rate, k_value, n_value, size, label_List):\n",
        "  result = 0.0\n",
        "  for i in range(0, size):\n",
        "    result = result + (x_list[i][n_value][0]*w_tmp[0][1]*v_tmp[1][k_value]*(sigmoid(h_list_tmp[i][0][0]) - label_List[i][0][0])*(sigmoid(z__list_tmp[i][1][0])*(1 - sigmoid(z__list_tmp[i][1][0])))*(sigmoid(y__list_tmp[i][k_value][0])*(1 - sigmoid(y__list_tmp[i][k_value][0]))))\n",
        "  result = result / size\n",
        "  result = result * learning_rate\n",
        "  final = u_tmp[k_value][n_value] - result\n",
        "\n",
        "  return final"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0vn3FkYXStur",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#이제 여기에다가\n",
        "\n",
        "#iteration에 따라\n",
        "#cost function 값을 배열에 저장하고\n",
        "#세타값을 업데이트 시키고(위의 세개 함수 호출)\n",
        "#업데이트 시킨값을 다시 cost함수에 넣고\n",
        "#업데이트 시키고\n",
        "#이것을 반복하는 함수를 짠다.\n",
        "\n",
        "#아직 해야하는 것 : h값에 따라 가장 큰값의 index로 L값정하는거 위에 추가해야함.\n",
        "\n",
        "iteration = 300\n",
        "l_r = 0.01\n",
        "\n",
        "cost_history = np.zeros(iteration)\n",
        "\n",
        "total_number_of_predictions = 6000\n",
        "accuracy_history = np.zeros(iteration)\n",
        "\n",
        "for t in range(0, iteration):\n",
        "  number_of_correct_predictions = 0\n",
        "  \n",
        "  print(t)\n",
        "\n",
        "  # cost_history[t] = objective_function(6000, l_list, h_list_tmp)\n",
        "  result = 0.0\n",
        "  for i in range(0, 6000):\n",
        "    for k in range(0, 10):\n",
        "      if h_list_tmp[i][k][0] <= 0:\n",
        "        print(h_list_tmp[i][k][0])\n",
        "      result = result + ((-1)*l_list[i][k]*math.log(h_list_tmp[i][k][0]) - (1-l_list[i][k])*math.log(1-h_list_tmp[i][k][0]))\n",
        "  result = result / 6000\n",
        "  cost_history[t] = result\n",
        "\n",
        "# (1 / (1 + math.exp(-y__list_tmp[i][m][0])))\n",
        "\n",
        "  for m in range(0, 10):\n",
        "    # print(m)\n",
        "    for n in range(0, 50):\n",
        "      # matrix_w[m][n] = w_gredient_descent(l_r, m, n, 6000, l_list)\n",
        "      result = 0.0\n",
        "      for i in range(0, 50):\n",
        "        result = result + (z_list_tmp[i][n][0]*((1 / (1 + math.exp(-h_list_tmp[i][m][0]))) - l_list[i][m][0]))\n",
        "      result = result / 50\n",
        "      result = result * l_r\n",
        "      final = w_tmp[m][n] - result\n",
        "      matrix_w[m][n] = final\n",
        "\n",
        "  for m in range(0, 50):\n",
        "    # print(m)\n",
        "    for n in range(0, 197):\n",
        "      # matrix_v[m][n] = v_gredient_descent(l_r, m, n, 6000, l_list)\n",
        "      result = 0.0\n",
        "      for i in range(0, 50):\n",
        "        result = result + (y_list_tmp[i][n][0]*w_tmp[0][m]*((1 / (1 + math.exp(-h_list_tmp[i][0][0]))) - l_list[i][0][0])*((1 / (1 + math.exp(-z__list_tmp[i][m][0])))*(1 - (1 / (1 + math.exp(-z__list_tmp[i][m][0]))))))\n",
        "      result = result / 50\n",
        "      result = result * l_r\n",
        "      final = v_tmp[m][n] - result\n",
        "      matrix_v[m][n] = final\n",
        "\n",
        "  for m in range(0, 197):\n",
        "    # print(m)\n",
        "    for n in range(0, 785):\n",
        "      # matrix_u[m][n] = u_gredient_descent(l_r, m, n, 6000, l_list)\n",
        "      result = 0.0\n",
        "      for i in range(0, 50):\n",
        "        result = result + (x_list[i][n][0]*w_tmp[0][1]*v_tmp[1][m]*((1 / (1 + math.exp(-h_list_tmp[i][0][0]))) - l_list[i][0][0])*((1 / (1 + math.exp(-z__list_tmp[i][1][0])))*(1 - (1 / (1 + math.exp(-z__list_tmp[i][1][0])))))*((1 / (1 + math.exp(-y__list_tmp[i][m][0])))*(1 - (1 / (1 + math.exp(-y__list_tmp[i][m][0]))))))\n",
        "      result = result / 50\n",
        "      result = result * l_r\n",
        "      final = u_tmp[m][n] - result\n",
        "      matrix_u[m][n] = final\n",
        "\n",
        "####################################################################\n",
        "\n",
        "  for k in range(0, 6000):\n",
        "  # print(i)\n",
        "    label       = list_label[k]\n",
        "    bias   = [1]       #bias 1\n",
        "    im_vector = np.concatenate((bias, list_image[:, k]), axis = None)\n",
        "\n",
        "    # print(type(list_image[: , k]))\n",
        "    # print(list_image[:, k])\n",
        "    # print(im_vector)\n",
        "    im_matrix   = im_vector.reshape((size_row*size_col + 1, 1))\n",
        "\n",
        "    y_ = np.dot(matrix_u, im_matrix)  #x와 가중치 u를 곱해서 y_를 만듦\n",
        "    y__list[k] = y_\n",
        "    \n",
        "    y_vector = np.ones((197,1)) #bias\n",
        "    for j in range (1, 197):\n",
        "      y_vector[j][0] = 1 / (1 + math.exp(-y_[j][0]))  #sigmoid 함수를 적용해서 y_에서 y를 만듦\n",
        "    \n",
        "    # print(y_vector) #시그모이드 값 확인\n",
        "    y   = y_vector.reshape((197, 1))\n",
        "    y_list[k] = y\n",
        "\n",
        "    z_  = np.dot(matrix_v, y)  #y와 가중치 v를 곱해서 z_를 만듦\n",
        "    z__list[k] = z_\n",
        "    z_vector = np.ones((50,1)) #bias\n",
        "    for j in range (1, 50):\n",
        "      z_vector[j][0] = 1 / (1 + math.exp(-z_[j][0]))  #sigmoid 함수를 적용해서 y_에서 y를 만듦\n",
        "\n",
        "    z   = z_vector.reshape((50, 1))\n",
        "    z_list[k] = z\n",
        "    \n",
        "    # print(z)\n",
        "\n",
        "    h_  = np.dot(matrix_w, z)  #z와 가중치 w를 곱해서 h_를 만듦\n",
        "    h__list[k] = h_\n",
        "    \n",
        "    h_vector = np.ones((10,1)) #bias\n",
        "    for j in range (0, 10):\n",
        "      h_vector[j][0] = 1 / (1 + math.exp(-h_[j][0]))  #sigmoid 함수를 적용해서 h_에서 h를 만듦\n",
        "\n",
        "    h   = h_vector.reshape((10, 1))\n",
        "    # tmp = h.ravel()\n",
        "    # max_value = max(tmp)\n",
        "    # print(max_value)\n",
        "    # max_index = tmp.index(max_value)\n",
        "    index, value = max(enumerate(h), key=operator.itemgetter(1))\n",
        "    h_list[k] = h\n",
        "    l_list[k][index] = 1\n",
        "    # print(l_list[i])\n",
        "    # print(h)\n",
        "    if index == label:\n",
        "      number_of_correct_predictions = number_of_correct_predictions + 1\n",
        "\n",
        "    y_list_tmp[k] = y_list[k]\n",
        "    z_list_tmp[k] = z_list[k]\n",
        "    h_list_tmp[k] = h_list[k]\n",
        "    y__list_tmp[k] = y__list[k]\n",
        "    z__list_tmp[k] = z__list[k]\n",
        "    h__list_tmp[k] = h__list[k]\n",
        "\n",
        "  accuracy_history[t] = number_of_correct_predictions/total_number_of_predictions\n",
        "  t = t + 1"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DcsYGj-cTUkN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#그 코드를 바탕으로 training loss plotting 하기.\n",
        "fig,ax = plt.subplots(figsize=(12,8))\n",
        "\n",
        "cost_tmp = np.zeros(300)\n",
        "\n",
        "for i in range(0, 300):\n",
        "  cost_tmp[i] = cost_history[i]\n",
        "\n",
        "\n",
        "# print(cost_history[1])\n",
        "\n",
        "ax.set_ylabel('J(Theta)')\n",
        "ax.set_xlabel('Iterations')\n",
        "_=ax.plot(range(300),cost_tmp,'b.', color = 'red')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2m0mvxqZUvHl",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#그 코드를 바탕으로 training loss plotting 하기.\n",
        "fig,ax = plt.subplots(figsize=(12,8))\n",
        "\n",
        "# print(cost_history[1])\n",
        "\n",
        "ac_tmp = np.zeros(300)\n",
        "\n",
        "for i in range(0, 300):\n",
        "  ac_tmp[i] = accuracy_history[i]\n",
        "\n",
        "ax.set_ylabel('accuracy')\n",
        "ax.set_xlabel('Iterations')\n",
        "_=ax.plot(range(300),ac_tmp,'b.', color = 'red')\n",
        "\n",
        "# print(ac_tmp)"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}