latent_vector_model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "with open('data.pickle', 'rb') as f:\n",
    "    base_params = pickle.load(f)\n",
    "# base_mui, base_bu, base_bi = best_set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Loading dataset\n",
    "df_train_full = pd.read_csv(\"./dataset/train.csv\")\n",
    "df_test = pd.read_csv(\"./dataset/test.csv\")\n",
    "df_submission = pd.read_csv(\"./dataset/dummy_submission.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Splitting into test and train data\n",
    "np.random.seed(5)\n",
    "indices = np.random.permutation(df_train_full.shape[0])\n",
    "training_idx, test_idx = indices[:int(0.85*len(df_train_full))], indices[int(0.85*len(df_train_full)):]\n",
    "df_train, df_csv = df_train_full.iloc[training_idx,:], df_train_full.iloc[test_idx,:]\n",
    "#Sorting for better data handling and clarity\n",
    "df_train=df_train.sort_values(by=['userId','movieId']).reset_index(drop=True)\n",
    "df_csv=df_csv.sort_values(by=['userId','movieId']).reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Baseline Prediction\n",
    "\n",
    "\n",
    "def minibatch(batch_no,df,batch_size):\n",
    "    '''Generates the batch for mini-batch gradient descent '''\n",
    "    if (batch_no+1)*batch_size > len(df):\n",
    "        minibatch_indices = indices[batch_size*(batch_no)::]\n",
    "    else:        \n",
    "        minibatch_indices = indices[batch_size*(batch_no):batch_size*(batch_no+1)]\n",
    "    X_train = df.iloc[minibatch_indices,:]\n",
    "    X_train = X_train.sort_values(by=['userId','movieId']).reset_index(drop=True)\n",
    "    return X_train\n",
    "\n",
    "def loss(parameters,df):\n",
    "    miu,bu,bi,pu,qi = parameters\n",
    "    loss = 0\n",
    "    for i in range(len(df)):\n",
    "        userid =  df.iloc[i,0]\n",
    "        movieid = df.iloc[i,1]\n",
    "        rating = miu+bu[userid]+bi[movieid]+np.dot(pu[userid],qi[movieid])\n",
    "        if rating>5:\n",
    "            rating = 5\n",
    "        if rating<0.5:\n",
    "            rating =0.5\n",
    "        loss = loss + (df.iloc[i,2]-(miu+bu[userid]+bi[movieid]+np.dot(pu[userid],qi[movieid])))**2\n",
    "    loss = loss/len(df)\n",
    "    return loss    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Latent_Training(step_size,dim,Epochs,batch_size,df,base_params):\n",
    "    base_mui, base_bu, base_bi = base_params\n",
    "    #Initialize values for bu,b'i,miu\n",
    "    #Since we are given there are 10k users and 10k movie id\n",
    "    pu = np.ones((10000,dim)) / dim\n",
    "    qi = np.ones((10000,dim)) / dim\n",
    "    #Taking miu from baseline\n",
    "    miu = base_mui\n",
    "    bu = base_bu\n",
    "    bi = base_bi\n",
    "    \n",
    "    num_batches = df.shape[0]//batch_size + 1\n",
    "    for epoch in range(Epochs):\n",
    "        for batch in range(num_batches):\n",
    "            bui = []\n",
    "            X = minibatch(batch,df,batch_size)\n",
    "            userid =  X.iloc[:,0]\n",
    "            movieid = X.iloc[:,1]\n",
    "            bui.append((miu+bu[userid]+bi[movieid]))\n",
    "            bui = np.ravel(np.array(bui).T)\n",
    "            pu_temp = pu\n",
    "            qi_temp = qi\n",
    "            for i in range(len(X)):\n",
    "                userid =  X.iloc[i,0]\n",
    "                movieid = X.iloc[i,1]\n",
    "                prod = np.dot(pu[userid],qi[movieid])\n",
    "                pu_temp[userid]  += step_size*qi[movieid]*(X.iloc[i,2]-bui[i]-prod)\n",
    "                qi_temp[movieid] += step_size*pu[userid]*(X.iloc[i,2]-bui[i]-prod)\n",
    "            pu = pu_temp\n",
    "            qi = qi_temp\n",
    "            \n",
    "            if batch %500 ==0:\n",
    "#                 print(pu,qi)\n",
    "                print(\"Epoch:{} Batch:{} ------ Training Error:{}\".format(epoch,batch,loss((miu,bu,bi,pu,qi),X)))\n",
    "    return pu, qi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:0 Batch:0 ------ Training Error:0.7388040818117134\n",
      "Epoch:0 Batch:500 ------ Training Error:0.7517068733107805\n",
      "Epoch:0 Batch:1000 ------ Training Error:0.6785573057103371\n",
      "Epoch:0 Batch:1500 ------ Training Error:0.6906565619176424\n",
      "Epoch:0 Batch:2000 ------ Training Error:0.7120948669959121\n",
      "Epoch:0 Batch:2500 ------ Training Error:0.7168174056862945\n",
      "Epoch:0 Batch:3000 ------ Training Error:0.6831573671906152\n",
      "Epoch:0 Batch:3500 ------ Training Error:0.7411393248829192\n",
      "Epoch:0 Batch:4000 ------ Training Error:0.6393509060116275\n",
      "Epoch:1 Batch:0 ------ Training Error:0.7370463905934963\n",
      "Epoch:1 Batch:500 ------ Training Error:0.749567433969079\n",
      "Epoch:1 Batch:1000 ------ Training Error:0.67611150406611\n",
      "Epoch:1 Batch:1500 ------ Training Error:0.6877412078322739\n",
      "Epoch:1 Batch:2000 ------ Training Error:0.7097882761978673\n",
      "Epoch:1 Batch:2500 ------ Training Error:0.7141817658693204\n",
      "Epoch:1 Batch:3000 ------ Training Error:0.682735260819086\n",
      "Epoch:1 Batch:3500 ------ Training Error:0.740733867424529\n",
      "Epoch:1 Batch:4000 ------ Training Error:0.6363635744831838\n",
      "Epoch:2 Batch:0 ------ Training Error:0.7341973699039002\n",
      "Epoch:2 Batch:500 ------ Training Error:0.7452145868599088\n",
      "Epoch:2 Batch:1000 ------ Training Error:0.6715050377344945\n",
      "Epoch:2 Batch:1500 ------ Training Error:0.6819539871315369\n",
      "Epoch:2 Batch:2000 ------ Training Error:0.7043698049716058\n",
      "Epoch:2 Batch:2500 ------ Training Error:0.7060569214571666\n",
      "Epoch:2 Batch:3000 ------ Training Error:0.6793844025208073\n",
      "Epoch:2 Batch:3500 ------ Training Error:0.7391178409100579\n",
      "Epoch:2 Batch:4000 ------ Training Error:0.6294402824065345\n",
      "Epoch:3 Batch:0 ------ Training Error:0.7245172720149595\n",
      "Epoch:3 Batch:500 ------ Training Error:0.7358220090274881\n",
      "Epoch:3 Batch:1000 ------ Training Error:0.656908411104035\n",
      "Epoch:3 Batch:1500 ------ Training Error:0.667197961760787\n",
      "Epoch:3 Batch:2000 ------ Training Error:0.6886220108459232\n",
      "Epoch:3 Batch:2500 ------ Training Error:0.6852779188766801\n",
      "Epoch:3 Batch:3000 ------ Training Error:0.6643133332303028\n",
      "Epoch:3 Batch:3500 ------ Training Error:0.7234896832731443\n",
      "Epoch:3 Batch:4000 ------ Training Error:0.616059214462246\n",
      "Epoch:4 Batch:0 ------ Training Error:0.7074359383451586\n",
      "Epoch:4 Batch:500 ------ Training Error:0.7236304265550144\n",
      "Epoch:4 Batch:1000 ------ Training Error:0.635001057350535\n",
      "Epoch:4 Batch:1500 ------ Training Error:0.6535877988624014\n",
      "Epoch:4 Batch:2000 ------ Training Error:0.6725605406176047\n",
      "Epoch:4 Batch:2500 ------ Training Error:0.6706024613737747\n",
      "Epoch:4 Batch:3000 ------ Training Error:0.6473284507080573\n",
      "Epoch:4 Batch:3500 ------ Training Error:0.7070439398188437\n",
      "Epoch:4 Batch:4000 ------ Training Error:0.6081162962227611\n",
      "Epoch:5 Batch:0 ------ Training Error:0.7011371958554007\n",
      "Epoch:5 Batch:500 ------ Training Error:0.7165897192002799\n",
      "Epoch:5 Batch:1000 ------ Training Error:0.6254004502121246\n",
      "Epoch:5 Batch:1500 ------ Training Error:0.6479177839369118\n",
      "Epoch:5 Batch:2000 ------ Training Error:0.6668831856779135\n",
      "Epoch:5 Batch:2500 ------ Training Error:0.668082429259222\n",
      "Epoch:5 Batch:3000 ------ Training Error:0.642383817722765\n",
      "Epoch:5 Batch:3500 ------ Training Error:0.7000219256668901\n",
      "Epoch:5 Batch:4000 ------ Training Error:0.6063905136715225\n",
      "Epoch:6 Batch:0 ------ Training Error:0.7001788697449132\n",
      "Epoch:6 Batch:500 ------ Training Error:0.7128742578085543\n",
      "Epoch:6 Batch:1000 ------ Training Error:0.6221736906977262\n",
      "Epoch:6 Batch:1500 ------ Training Error:0.6457722941160096\n",
      "Epoch:6 Batch:2000 ------ Training Error:0.664454701110466\n",
      "Epoch:6 Batch:2500 ------ Training Error:0.6674901992502513\n",
      "Epoch:6 Batch:3000 ------ Training Error:0.6410401019663671\n",
      "Epoch:6 Batch:3500 ------ Training Error:0.69722316564012\n",
      "Epoch:6 Batch:4000 ------ Training Error:0.6063934609188745\n",
      "Epoch:7 Batch:0 ------ Training Error:0.6998693494773727\n",
      "Epoch:7 Batch:500 ------ Training Error:0.7108587152371465\n",
      "Epoch:7 Batch:1000 ------ Training Error:0.6207713233674383\n",
      "Epoch:7 Batch:1500 ------ Training Error:0.6447963038343107\n",
      "Epoch:7 Batch:2000 ------ Training Error:0.6632977158607891\n",
      "Epoch:7 Batch:2500 ------ Training Error:0.6671952112262787\n",
      "Epoch:7 Batch:3000 ------ Training Error:0.6403188922962642\n",
      "Epoch:7 Batch:3500 ------ Training Error:0.695827502979891\n",
      "Epoch:7 Batch:4000 ------ Training Error:0.6065852159292836\n",
      "Epoch:8 Batch:0 ------ Training Error:0.6996017129089322\n",
      "Epoch:8 Batch:500 ------ Training Error:0.7097908905315411\n",
      "Epoch:8 Batch:1000 ------ Training Error:0.6200070631085052\n",
      "Epoch:8 Batch:1500 ------ Training Error:0.6442371428361298\n",
      "Epoch:8 Batch:2000 ------ Training Error:0.6626344619529477\n",
      "Epoch:8 Batch:2500 ------ Training Error:0.6669883721014255\n",
      "Epoch:8 Batch:3000 ------ Training Error:0.6397947795389638\n",
      "Epoch:8 Batch:3500 ------ Training Error:0.6949690455493907\n",
      "Epoch:8 Batch:4000 ------ Training Error:0.6067223866123266\n",
      "Epoch:9 Batch:0 ------ Training Error:0.6993354431651249\n",
      "Epoch:9 Batch:500 ------ Training Error:0.7092189799741755\n",
      "Epoch:9 Batch:1000 ------ Training Error:0.6195257485752185\n",
      "Epoch:9 Batch:1500 ------ Training Error:0.6438753344924052\n",
      "Epoch:9 Batch:2000 ------ Training Error:0.6621791093766539\n",
      "Epoch:9 Batch:2500 ------ Training Error:0.6668255003528637\n",
      "Epoch:9 Batch:3000 ------ Training Error:0.6393789829505441\n",
      "Epoch:9 Batch:3500 ------ Training Error:0.6943624489476719\n",
      "Epoch:9 Batch:4000 ------ Training Error:0.6068038915323317\n"
     ]
    }
   ],
   "source": [
    "np.random.seed(5)\n",
    "indices = np.random.permutation(df_train.shape[0])\n",
    "\n",
    "#Hyperparameters\n",
    "step_size = 0.01\n",
    "dim = 20\n",
    "Epochs = 10\n",
    "batch_size = 1024  \n",
    "\n",
    "#Regularization parameter tuning\n",
    "pu, qi = Latent_Training(step_size,dim,Epochs,batch_size,df_train, base_params)\n",
    "best_pq = (pu,qi)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "with open('data_pq.pickle', 'wb') as f:\n",
    "    pickle.dump(best_pq, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.6508727320818251 0.6601834151059541\n"
     ]
    }
   ],
   "source": [
    "base_mui, base_bu, base_bi = base_params\n",
    "best_params = (base_mui, base_bu, base_bi, pu, qi)\n",
    "train_loss = loss(best_params,df_train)\n",
    "val_loss = loss(best_params,df_csv) \n",
    "print(train_loss, val_loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Submission\n",
    "predictions = np.zeros(len(df_test))\n",
    "for i in range(len(df_test)):\n",
    "    userid =  df_test.iloc[i,0]\n",
    "    movieid = df_test.iloc[i,1]\n",
    "    rating = float(\"{0:.1f}\".format(base_mui+base_bu[userid]+base_bi[movieid]+np.dot(pu[userid],qi[movieid])))\n",
    "\n",
    "    if rating>5:\n",
    "        rating = 5\n",
    "    if rating<0.5:\n",
    "        rating =0.5\n",
    "    predictions[i] = rating\n",
    "df_submission.Prediction = predictions\n",
    "df_submission.to_csv('./Submission.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# baseline_train_loss = 0.7006116025819583 \n",
    "# baseline_val_loss = 0.7062281886121021"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}