create_data_matrix.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pickle\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(31, 2.5), (1029, 3.0), (1061, 3.0), (1129, 2.0), (1172, 4.0), (1263, 2.0), (1287, 2.0), (1293, 2.0), (1339, 3.5), (1343, 2.0), (1371, 2.5), (1405, 1.0), (1953, 4.0), (2105, 4.0), (2150, 3.0), (2193, 2.0), (2294, 2.0), (2455, 2.5), (2968, 1.0), (3671, 3.0)]\n"
     ]
    }
   ],
   "source": [
    "#we're looking to populate these 4 dictionaries\n",
    "movieId_movieName = {}\n",
    "movieId_movieCol  = {}\n",
    "userId_userRow    = {}\n",
    "userId_rating     = {}\n",
    "\n",
    "#for reference\n",
    "movieId_isRated   = {}\n",
    "\n",
    "#then populate a data matrix based on the userId_rating dictionary\n",
    "\n",
    "\n",
    "'''\n",
    "Read Basic Movie Info\n",
    "'''\n",
    "\n",
    "path     = 'data'\n",
    "filename = 'movies.csv'\n",
    "\n",
    "\n",
    "dataFrame = pd.read_csv('{}/{}'.format(path,filename))\n",
    "\n",
    "for i in range(len(dataFrame['movieId'])):\n",
    "    movieId = dataFrame['movieId'][i]\n",
    "    \n",
    "    movieId_movieName[movieId] = dataFrame['title'][i]    \n",
    "    movieId_isRated[movieId]    = 0\n",
    "\n",
    "   \n",
    "'''\n",
    "Read in the ratings\n",
    "'''\n",
    "path     = 'data'\n",
    "filename = 'ratings.csv'\n",
    "\n",
    "\n",
    "dataFrame = pd.read_csv('{}/{}'.format(path,filename))\n",
    "\n",
    "for i in range(len(dataFrame)):\n",
    "    userId  = dataFrame['userId'][i]\n",
    "    movieId = dataFrame['movieId'][i]\n",
    "    rating  = dataFrame['rating'][i]\n",
    "    \n",
    "    if userId not in userId_rating.keys():\n",
    "        userId_rating[userId] = [(movieId, rating)]\n",
    "    else:\n",
    "        userId_rating[userId].append((movieId, rating))\n",
    "    \n",
    "    movieId_isRated[movieId] = 1\n",
    "        \n",
    "print(userId_rating[1])\n",
    "    \n",
    "\n",
    "for movieId, isRated in movieId_isRated.items():\n",
    "    if isRated == 0:\n",
    "        del movieId_movieName[movieId]\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(671, 9066)\n",
      "Number of ratings = 100004\n",
      "Total entries = 6083286\n",
      "Sparsity = 1.6439141608663477%\n"
     ]
    }
   ],
   "source": [
    "'''\n",
    "Create the row-column data\n",
    "\n",
    "the rows and columns will have the entries for Ids in sorted order.\n",
    "Therefore:\n",
    "    the ith row    in the data matrix will be the ith key in sorted movieIds\n",
    "    the jth column in the data matrix will be the jth key in sorted userIds\n",
    "'''\n",
    "userId_userRow\n",
    "movieId_movieCol\n",
    "\n",
    "i = 0\n",
    "for movieId in sorted(movieId_movieName):\n",
    "    movieId_movieCol[movieId] = i\n",
    "    i+=1\n",
    "\n",
    "i=0\n",
    "for userId in sorted(userId_rating):\n",
    "    userId_userRow[userId] = i\n",
    "    i+=1\n",
    "\n",
    "m = len(userId_userRow.keys())\n",
    "n = len(movieId_movieCol.keys())\n",
    "A = np.zeros((m,n))\n",
    "\n",
    "\n",
    "    \n",
    "print(A.shape)\n",
    "for userId, ratings in userId_rating.items():\n",
    "    for rating in ratings:\n",
    "        movieId   = rating[0]\n",
    "        score     = rating[1]\n",
    "        \n",
    "        if (userId in userId_userRow and movieId in movieId_movieCol):\n",
    "            i = userId_userRow[userId]\n",
    "\n",
    "            j = movieId_movieCol[movieId]\n",
    "            A[i,j] = score\n",
    "\n",
    "ratingCount = 0\n",
    "for i in range(m):\n",
    "    for j in range(n):\n",
    "        if (A[i][j] != 0):\n",
    "#             if(ratingCount < 20):\n",
    "#                 print(A[i][j])\n",
    "            ratingCount += 1\n",
    "\n",
    "\n",
    "print('Number of ratings = {}'.format(ratingCount))\n",
    "print('Total entries = {}'.format(m*n))\n",
    "print('Sparsity = {}%'.format(ratingCount*100/(m*n)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(671, 9066)\n"
     ]
    }
   ],
   "source": [
    "'''\n",
    "write the relevant dictionaries and matrix to a file\n",
    "'''\n",
    "# movieId_movieName\n",
    "# movieId_movieCol\n",
    "# userId_userRow\n",
    "# userId_rating\n",
    "# A\n",
    "\n",
    "d = {'movieId_movieName': movieId_movieName,\n",
    "     'movieId_movieCol' : movieId_movieCol,\n",
    "     'userId_userRow'   : userId_userRow,\n",
    "     'userId_rating'    : userId_rating }\n",
    "pickle.dump(A, open('data/data_matrix.p', 'wb'))\n",
    "pickle.dump(d, open('data/data_dicts.p', 'wb'))\n",
    "print (A.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}