tf_logreg_regularize.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from six.moves import cPickle as pickle\n",
    "import random\n",
    "import functools\n",
    "from functools import reduce"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "image_size = 28\n",
    "input_num_units=28*28\n",
    "num_labels = 10\n",
    "output_num_units=num_labels\n",
    "learning_rate=0.5\n",
    "num_steps =501\n",
    "num_train_samples = 10000\n",
    "beta=0.01"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training set (200000, 28, 28) (200000,)\n",
      "Validation set (10000, 28, 28) (10000,)\n",
      "Test set (10000, 28, 28) (10000,)\n"
     ]
    }
   ],
   "source": [
    "pickle_file = 'notMNIST.pickle'\n",
    "with open(pickle_file, 'rb') as f:\n",
    "    save = pickle.load(f)\n",
    "    train_dataset = save['train_dataset']\n",
    "    train_labels = save['train_labels']\n",
    "    valid_dataset = save['valid_dataset']\n",
    "    valid_labels = save['valid_labels']\n",
    "    test_dataset = save['test_dataset']\n",
    "    test_labels = save['test_labels']\n",
    "    del save  # hint to help gc free up memory\n",
    "    print('Training set', train_dataset.shape, train_labels.shape)\n",
    "    print('Validation set', valid_dataset.shape, valid_labels.shape)\n",
    "    print('Test set', test_dataset.shape, test_labels.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training set (200000, 784) (200000, 10)\n",
      "Validation set (10000, 784) (10000, 10)\n",
      "Test set (10000, 784) (10000, 10)\n"
     ]
    }
   ],
   "source": [
    "def reformat(dataset, labels):\n",
    "    dataset = dataset.reshape((-1, image_size * image_size)).astype(np.float32)\n",
    "    labels = (np.arange(num_labels) == labels[:,None]).astype(np.float32)#Concise 1-Hot Encoding in NUMPY\n",
    "    return dataset, labels\n",
    "train_dataset, train_labels = reformat(train_dataset, train_labels)\n",
    "valid_dataset, valid_labels = reformat(valid_dataset, valid_labels)\n",
    "test_dataset, test_labels = reformat(test_dataset, test_labels)\n",
    "print('Training set', train_dataset.shape, train_labels.shape)\n",
    "print('Validation set', valid_dataset.shape, valid_labels.shape)\n",
    "print('Test set', test_dataset.shape, test_labels.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def getSampleIndexes(population,sample_sz):\n",
    "    if(sample_sz>population.shape[0]):\n",
    "        return np.array(range(population.shape[0]))\n",
    "    rand_indexes=random.sample(range(population.shape[0]),sample_sz)\n",
    "    return population[rand_indexes,:],rand_indexes\n",
    "def accuracy(predictions, labels):\n",
    "    return (100.0 * np.sum(np.argmax(predictions, 1) == np.argmax(labels, 1))/ predictions.shape[0])\n",
    "def isEqual(nparr1,nparr2):\n",
    "    totDims=len(nparr1.shape)\n",
    "    if np.sum([nparr1.shape[i]==nparr2.shape[i] for i in range(totDims)])!=totDims:\n",
    "        return False\n",
    "    eqItems=nparr1==nparr2\n",
    "    if np.sum(eqItems)==reduce(lambda x,y:x*y,nparr1.shape):\n",
    "        return True\n",
    "    else:\n",
    "        return False\n",
    "def apply_reg_opt(weight,bias,train_x,train_y,regularizer):\n",
    "    weighted_sum = tf.matmul(train_x, weight) + bias \n",
    "    cost=tf.nn.softmax_cross_entropy_with_logits(labels=train_y, logits=weighted_sum)\n",
    "    cost=cost+regularizer\n",
    "    loss_fun = tf.reduce_mean(cost)\n",
    "    optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss_fun)\n",
    "    return optimizer,loss_fun\n",
    "def predict_values(x,weights,biases):\n",
    "    return tf.nn.softmax(tf.matmul(x,weights)+biases)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#Logistic Regression : Declaring the Model as Computational Graph\n",
    "graph_reg = tf.Graph()\n",
    "with graph_reg.as_default():\n",
    "    train_dataset_tf = tf.placeholder(tf.float32, [None, input_num_units])\n",
    "    train_labels_tf = tf.placeholder(tf.float32, [None, num_labels])\n",
    "    valid_dataset_tf = tf.placeholder(tf.float32, [None, input_num_units])\n",
    "    test_dataset_tf = tf.placeholder(tf.float32, [None, input_num_units])\n",
    "    \n",
    "    weights_l2 = tf.Variable(tf.truncated_normal([input_num_units, num_labels]))\n",
    "    biases_l2 = tf.Variable(tf.zeros([num_labels]))\n",
    "    l2_penalty=beta*tf.nn.l2_loss(weights_l2)\n",
    "    optimizer_l2,loss_fun=apply_reg_opt(weights_l2,biases_l2,train_dataset_tf,train_labels_tf,l2_penalty)\n",
    "    \n",
    "    train_prediction_l2 = predict_values(train_dataset_tf,weights_l2,biases_l2)\n",
    "    valid_prediction_l2 = predict_values(valid_dataset_tf,weights_l2,biases_l2)\n",
    "    test_prediction_l2 = predict_values(test_dataset_tf,weights_l2,biases_l2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initialized\n",
      "Computational Graph : Validation Passed!\n",
      "Loss at step 0: 43.515495\n",
      "Accuracy ~~~~~~~~~~~~> 14.78 18.27 19.26\n",
      ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "Loss at step 100: 11.849002\n",
      "Accuracy ~~~~~~~~~~~~> 73.3 72.7 81.0\n",
      ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "Loss at step 200: 4.498791\n",
      "Accuracy ~~~~~~~~~~~~> 77.99 76.79 85.4\n",
      ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "Loss at step 300: 1.995128\n",
      "Accuracy ~~~~~~~~~~~~> 80.67 79.63 88.02\n",
      ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "Loss at step 400: 1.162605\n",
      "Accuracy ~~~~~~~~~~~~> 82.11 80.77 89.02\n",
      ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n",
      "Loss at step 500: 0.875537\n",
      "Accuracy ~~~~~~~~~~~~> 82.31 81.25 89.47\n",
      ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n"
     ]
    }
   ],
   "source": [
    "def getFeedDict(train_x,train_y):\n",
    "        return {train_dataset_tf:train_x,train_labels_tf:train_y,valid_dataset_tf:valid_dataset,\n",
    "              test_dataset_tf:test_dataset}\n",
    "def validate_tf_comput_graph(session,tf_feed_dict):\n",
    "    _=session.run([optimizer_l2],feed_dict=tf_feed_dict)\n",
    "    train_pred_1=session.run(train_prediction_l2,feed_dict=tf_feed_dict)\n",
    "    train_pred_2=session.run(train_prediction_l2,feed_dict=tf_feed_dict)\n",
    "    if isEqual(train_pred_1,train_pred_2):\n",
    "        print(\"Computational Graph : Validation Passed!\")\n",
    "    else:\n",
    "        print(\"Computational Graph : Validation Failed!\")\n",
    "with tf.Session(graph=graph_reg) as session:\n",
    "    tf.global_variables_initializer().run()\n",
    "    print('Initialized')\n",
    "    train_x,rand_indexes=getSampleIndexes(train_dataset,num_train_samples)\n",
    "    train_y=train_labels[rand_indexes]\n",
    "    validate_tf_comput_graph(session,getFeedDict(train_x,train_y))\n",
    "    for step in range(num_steps):\n",
    "        train_x,rand_indexes=getSampleIndexes(train_dataset,num_train_samples)\n",
    "        train_y=train_labels[rand_indexes]\n",
    "        _,loss,train_pred_l2=session.run([optimizer_l2,loss_fun, train_prediction_l2],\n",
    "                                      feed_dict=getFeedDict(train_x,train_y))\n",
    "        if step%100==0:\n",
    "            print('Loss at step %d: %f' % (step, loss))\n",
    "            valid_pred_l2,test_pred_l2=session.run([valid_prediction_l2,test_prediction_l2],\n",
    "                                                   feed_dict=getFeedDict(train_x,train_y))\n",
    "            train_acc_l2=accuracy(train_pred_l2, train_y)\n",
    "            valid_acc_l2=accuracy(valid_pred_l2, valid_labels)\n",
    "            test_acc_l2=accuracy(test_pred_l2,test_labels)\n",
    "            print(\"Accuracy ~~~~~~~~~~~~>\",train_acc_l2,valid_acc_l2,test_acc_l2)\n",
    "            print(\">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tensorflow",
   "language": "python",
   "name": "tensorflow"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}