Polysem.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "\n",
    "# Global variables.\n",
    "NUM_LABELS = 2    # The number of labels.\n",
    "\n",
    "# flag_name is second_digit, half_ones, sum_of_digits\n",
    "flag_name = 'half_ones'\n",
    "flag_length = '7'\n",
    "flag_epochs = 10\n",
    "flag_hidden = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def extract_data(filename):\n",
    "    # Arrays to hold the labels and feature vectors.\n",
    "    labels = []\n",
    "    fvecs = []\n",
    "\n",
    "    # Iterate over the rows, splitting the label from the features. Convert labels\n",
    "    # to integers and features to floats.\n",
    "    f = open(filename, 'r')\n",
    "    \n",
    "    for line in f:\n",
    "        row = line.strip('\\n').split(\",\")\n",
    "        labels.append(int(row[0]))\n",
    "        fvecs.append([float(x) for x in row[1:]])\n",
    "        #print(\"Vector length \" + str(len([float(x) for x in row[1:]])))\n",
    "\n",
    "    # Convert the array of float arrays into a numpy float matrix.\n",
    "    fvecs_np = np.matrix(fvecs).astype(np.float32)\n",
    "\n",
    "    # Convert the array of int labels into a numpy array.\n",
    "    labels_np = np.array(labels).astype(dtype=np.uint8)\n",
    "\n",
    "    # Convert the int numpy array into a one-hot matrix.\n",
    "    labels_onehot = (np.arange(NUM_LABELS) == labels_np[:, None]).astype(np.float32)\n",
    "\n",
    "    # Return a pair of the feature matrix and the one-hot label matrix.\n",
    "    return fvecs_np,labels_onehot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def init_weights(shape, init_method='xavier', xavier_params = (None, None)):\n",
    "    if init_method == 'zeros':\n",
    "        return tf.Variable(tf.zeros(shape, dtype=tf.float32))\n",
    "    elif init_method == 'uniform':\n",
    "        return tf.Variable(tf.random_normal(shape, stddev=0.01, dtype=tf.float32))\n",
    "    else: #xavier\n",
    "        (fan_in, fan_out) = xavier_params\n",
    "        low = -4*np.sqrt(6.0/(fan_in + fan_out)) # {sigmoid:4, tanh:1} \n",
    "        high = 4*np.sqrt(6.0/(fan_in + fan_out))\n",
    "        return tf.Variable(tf.random_uniform(shape, minval=low, maxval=high, dtype=tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initialized!\n",
      "\n",
      "Training.\n",
      "Accuracy at step 0: 0.688525\n",
      "Accuracy at step 10: 0.704918\n",
      "Accuracy at step 20: 0.934426\n",
      "Accuracy at step 30: 0.934426\n",
      "Accuracy at step 40: 0.95082\n",
      "Accuracy at step 50: 1.0\n",
      "Accuracy at step 60: 1.0\n",
      "Accuracy at step 70: 1.0\n",
      "Accuracy at step 80: 1.0\n",
      "Accuracy at step 90: 1.0\n",
      "Accuracy at step 100: 1.0\n",
      "Accuracy at step 110: 1.0\n",
      "Accuracy at step 120: 1.0\n",
      "Accuracy at step 130: 1.0\n",
      "Accuracy at step 140: 1.0\n",
      "Accuracy at step 150: 1.0\n",
      "Accuracy at step 160: 1.0\n",
      "Accuracy at step 170: 1.0\n",
      "Accuracy at step 180: 1.0\n",
      "Accuracy at step 190: 1.0\n",
      "Accuracy at step 200: 1.0\n",
      "Accuracy at step 210: 1.0\n",
      "Accuracy at step 220: 1.0\n",
      "Accuracy at step 230: 1.0\n",
      "Accuracy at step 240: 1.0\n",
      "Accuracy at step 250: 1.0\n",
      "Accuracy at step 260: 1.0\n",
      "Accuracy at step 270: 1.0\n",
      "Accuracy at step 280: 1.0\n",
      "Accuracy at step 290: 1.0\n",
      "Accuracy at step 300: 1.0\n",
      "Accuracy at step 310: 1.0\n",
      "Accuracy at step 320: 1.0\n",
      "Accuracy at step 330: 1.0\n",
      "Accuracy at step 340: 1.0\n",
      "Accuracy at step 350: 1.0\n",
      "Accuracy at step 360: 1.0\n",
      "Accuracy at step 370: 1.0\n",
      "Accuracy at step 380: 1.0\n",
      "Accuracy at step 390: 1.0\n",
      "Accuracy at step 400: 1.0\n",
      "Accuracy at step 410: 1.0\n",
      "Accuracy at step 420: 1.0\n",
      "Accuracy at step 430: 1.0\n",
      "Accuracy at step 440: 1.0\n",
      "Accuracy at step 450: 1.0\n",
      "Accuracy at step 460: 1.0\n",
      "Accuracy at step 470: 1.0\n",
      "Accuracy at step 480: 1.0\n",
      "Accuracy at step 490: 1.0\n",
      "Accuracy at step 500: 1.0\n",
      "Accuracy at step 510: 1.0\n",
      "Accuracy at step 520: 1.0\n",
      "Accuracy at step 530: 1.0\n",
      "Accuracy at step 540: 1.0\n",
      "Accuracy at step 550: 1.0\n",
      "Accuracy at step 560: 1.0\n",
      "Accuracy at step 570: 1.0\n",
      "Accuracy at step 580: 1.0\n",
      "Accuracy at step 590: 1.0\n",
      "Accuracy at step 600: 1.0\n",
      "Accuracy at step 610: 1.0\n",
      "Accuracy at step 620: 1.0\n",
      "Accuracy at step 630: 1.0\n",
      "Accuracy at step 640: 1.0\n",
      "Accuracy at step 650: 1.0\n",
      "Accuracy at step 660: 1.0\n",
      "1.0\n"
     ]
    }
   ],
   "source": [
    "# For tensorboard\n",
    "sess = tf.InteractiveSession()\n",
    "    \n",
    "# Get the data.\n",
    "train_data_filename = \"data/outfile-length\" + flag_length + \"-train-\" + flag_name + \".csv\"\n",
    "test_data_filename = \"data/outfile-length\" + flag_length + \"-eval-\" + flag_name + \".csv\"\n",
    "\n",
    "# Extract it into numpy arrays.\n",
    "train_data,train_labels = extract_data(train_data_filename)\n",
    "test_data, test_labels = extract_data(test_data_filename)\n",
    "\n",
    "# Get the shape of the training data.\n",
    "train_size,num_features = train_data.shape\n",
    "\n",
    "# Get the number of epochs for training.\n",
    "num_epochs = flag_epochs\n",
    "\n",
    "# Get the size of layer one.\n",
    "num_hidden = flag_hidden\n",
    " \n",
    "# This is where training samples and labels are fed to the graph.\n",
    "# These placeholder nodes will be fed a batch of training data at each\n",
    "# training step using the {feed_dict} argument to the Run() call below.\n",
    "x = tf.placeholder(\"float\", shape=[None, num_features])\n",
    "y_ = tf.placeholder(\"float\", shape=[None, NUM_LABELS])\n",
    "        \n",
    "# For the test data, hold the entire dataset in one constant node.\n",
    "test_data_node = tf.constant(test_data)\n",
    "\n",
    "# Define and initialize the network.\n",
    "\n",
    "# Initialize the hidden weights and biases.\n",
    "w_hidden = init_weights(\n",
    "    [num_features, num_hidden],\n",
    "    'xavier',\n",
    "    xavier_params=(num_features, num_hidden))\n",
    "\n",
    "b_hidden = init_weights([1,num_hidden],'zeros')\n",
    "\n",
    "# The hidden layer.\n",
    "hidden = tf.nn.tanh(tf.matmul(x,w_hidden) + b_hidden)\n",
    "# subst tf.nn.relu here for rectified linear units, the original is tanh\n",
    "# if you use relu, you should probably switch out `xavier` below for `uniform`\n",
    "\n",
    "# Initialize the output weights and biases.\n",
    "w_out = init_weights(\n",
    "    [num_hidden, NUM_LABELS],\n",
    "    'xavier',\n",
    "    xavier_params=(num_hidden, NUM_LABELS))\n",
    "    \n",
    "b_out = init_weights([1,NUM_LABELS],'zeros')\n",
    "\n",
    "# The output layer.\n",
    "y = tf.nn.softmax(tf.matmul(hidden, w_out) + b_out)\n",
    "    \n",
    "# Stuff for tensorboard\n",
    "w_hist = tf.histogram_summary(\"weights\", w_hidden)\n",
    "b_hist = tf.histogram_summary(\"biases\", b_hidden)\n",
    "y_hist = tf.histogram_summary(\"y\", y)\n",
    "    \n",
    "# Optimization.\n",
    "cross_entropy = -tf.reduce_sum(y_*tf.log(y))\n",
    "train_step = tf.train.GradientDescentOptimizer(0.01).minimize(cross_entropy)\n",
    "    \n",
    "# Evaluation.\n",
    "correct_prediction = tf.equal(tf.argmax(y,1), tf.argmax(y_,1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, \"float\"))\n",
    "\n",
    "accuracy_summary = tf.scalar_summary(\"accuracy\", accuracy)\n",
    "\t\n",
    "# More tensorboard stuff: merge all the summaries and write them out\n",
    "merged = tf.merge_all_summaries()\n",
    "summary_filename = \"logs/\" + flag_name + \"-length\" + flag_length\n",
    "writer = tf.train.SummaryWriter(summary_filename, sess.graph)\n",
    "    \n",
    "# Run all the initializers to prepare the trainable parameters.\n",
    "tf.initialize_all_variables().run()\n",
    "    \t\n",
    "print('Initialized!')\n",
    "print\n",
    "print('Training.')\n",
    "    \t\n",
    "# TODO: at the moment TensorBoard is disabled\n",
    "# Iterate and train.\n",
    "for step in range(num_epochs * train_size):\n",
    "    if step % 10 == 0: # just test, no learning\n",
    "        feed = {x: test_data, y_: test_labels}\n",
    "        result=sess.run(accuracy,feed_dict=feed)\n",
    "        #summary_str = result[0]\n",
    "        #acc = result[0]\n",
    "        #writer.add_summary(summary_str, step)\n",
    "        print(\"Accuracy at step %s: %s\" % (step, result))\n",
    "    else: # actually do learning\n",
    "        offset = step % train_size\n",
    "        batch_data = train_data[offset:(offset + 1), :]\n",
    "        batch_labels = train_labels[offset:(offset + 1)]\n",
    "        feed={x: batch_data, y_: batch_labels}\n",
    "        sess.run(train_step, feed_dict=feed)\n",
    "    \n",
    "print(accuracy.eval({x: test_data, y_: test_labels}))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [Root]",
   "language": "python",
   "name": "Python [Root]"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}