hybridContentBasedFiltering.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Hybrid System"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this work, the content-based recommender is combined with the SVD++ rating predictor to give high rating recommendations based on the particular user."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import joblib\n",
    "from surprise import Reader, Dataset, SVDpp\n",
    "from surprise.model_selection import KFold\n",
    "from surprise import accuracy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Similarity Matrix Created by Content-Based Recommender System"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>genres</th>\n",
       "      <th>title</th>\n",
       "      <th>cast</th>\n",
       "      <th>keywords</th>\n",
       "      <th>director</th>\n",
       "      <th>desc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>862</td>\n",
       "      <td>['animation', 'comedy', 'family']</td>\n",
       "      <td>Toy Story</td>\n",
       "      <td>['tomhanks', 'timallen', 'donrickles']</td>\n",
       "      <td>['jealousi', 'toy', 'boy', 'friendship', 'frie...</td>\n",
       "      <td>['johnlasseter']</td>\n",
       "      <td>animation comedy family tomhanks timallen donr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>8844</td>\n",
       "      <td>['adventure', 'fantasy', 'family']</td>\n",
       "      <td>Jumanji</td>\n",
       "      <td>['robinwilliams', 'jonathanhyde', 'kirstendunst']</td>\n",
       "      <td>['disappear', 'basedonchildrensbook']</td>\n",
       "      <td>['joejohnston']</td>\n",
       "      <td>adventure fantasy family robinwilliams jonatha...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>15602</td>\n",
       "      <td>['romance', 'comedy']</td>\n",
       "      <td>Grumpier Old Men</td>\n",
       "      <td>['waltermatthau', 'jacklemmon', 'annmargret']</td>\n",
       "      <td>['fish', 'bestfriend', 'duringcreditssting']</td>\n",
       "      <td>['howarddeutch']</td>\n",
       "      <td>romance comedy waltermatthau jacklemmon annmar...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>31357</td>\n",
       "      <td>['comedy', 'drama', 'romance']</td>\n",
       "      <td>Waiting to Exhale</td>\n",
       "      <td>['whitneyhouston', 'angelabassett', 'lorettade...</td>\n",
       "      <td>['basedonnovel', 'interracialrelationship', 's...</td>\n",
       "      <td>['forestwhitaker']</td>\n",
       "      <td>comedy drama romance whitneyhouston angelabass...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>11862</td>\n",
       "      <td>['comedy']</td>\n",
       "      <td>Father of the Bride Part II</td>\n",
       "      <td>['stevemartin', 'dianekeaton', 'martinshort']</td>\n",
       "      <td>['babi', 'midlifecrisi', 'confid', 'age', 'dau...</td>\n",
       "      <td>['charlesshyer']</td>\n",
       "      <td>comedy stevemartin dianekeaton martinshort bab...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      id                              genres                        title  \\\n",
       "0    862   ['animation', 'comedy', 'family']                    Toy Story   \n",
       "1   8844  ['adventure', 'fantasy', 'family']                      Jumanji   \n",
       "2  15602               ['romance', 'comedy']             Grumpier Old Men   \n",
       "3  31357      ['comedy', 'drama', 'romance']            Waiting to Exhale   \n",
       "4  11862                          ['comedy']  Father of the Bride Part II   \n",
       "\n",
       "                                                cast  \\\n",
       "0             ['tomhanks', 'timallen', 'donrickles']   \n",
       "1  ['robinwilliams', 'jonathanhyde', 'kirstendunst']   \n",
       "2      ['waltermatthau', 'jacklemmon', 'annmargret']   \n",
       "3  ['whitneyhouston', 'angelabassett', 'lorettade...   \n",
       "4      ['stevemartin', 'dianekeaton', 'martinshort']   \n",
       "\n",
       "                                            keywords            director  \\\n",
       "0  ['jealousi', 'toy', 'boy', 'friendship', 'frie...    ['johnlasseter']   \n",
       "1              ['disappear', 'basedonchildrensbook']     ['joejohnston']   \n",
       "2       ['fish', 'bestfriend', 'duringcreditssting']    ['howarddeutch']   \n",
       "3  ['basedonnovel', 'interracialrelationship', 's...  ['forestwhitaker']   \n",
       "4  ['babi', 'midlifecrisi', 'confid', 'age', 'dau...    ['charlesshyer']   \n",
       "\n",
       "                                                desc  \n",
       "0  animation comedy family tomhanks timallen donr...  \n",
       "1  adventure fantasy family robinwilliams jonatha...  \n",
       "2  romance comedy waltermatthau jacklemmon annmar...  \n",
       "3  comedy drama romance whitneyhouston angelabass...  \n",
       "4  comedy stevemartin dianekeaton martinshort bab...  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open(\"hybrid/svdpp.joblib\", \"rb\") as f:\n",
    "    svd = joblib.load(f)\n",
    "    \n",
    "dists = np.load(\"contentSim/metaSim.npy\")\n",
    "md = pd.read_csv(\"contentSim/metaFeatures.csv\")\n",
    "md.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Rating Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "ratings = pd.read_csv(\"../data/movies/ratings_small.csv\")\n",
    "\n",
    "reader = Reader(rating_scale=(1, 5))\n",
    "data = Dataset.load_from_df(ratings[[\"userId\", \"movieId\", \"rating\"]], reader)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train a Rating Predictor Using SVD++"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Fold 1\n",
      " processing epoch 0\n",
      " processing epoch 1\n",
      " processing epoch 2\n",
      " processing epoch 3\n",
      " processing epoch 4\n",
      " processing epoch 5\n",
      " processing epoch 6\n",
      " processing epoch 7\n",
      " processing epoch 8\n",
      " processing epoch 9\n",
      " processing epoch 10\n",
      " processing epoch 11\n",
      " processing epoch 12\n",
      " processing epoch 13\n",
      " processing epoch 14\n",
      " processing epoch 15\n",
      " processing epoch 16\n",
      " processing epoch 17\n",
      " processing epoch 18\n",
      " processing epoch 19\n",
      "MSE: 0.8085\n",
      "\n",
      "Fold 2\n",
      " processing epoch 0\n",
      " processing epoch 1\n",
      " processing epoch 2\n",
      " processing epoch 3\n",
      " processing epoch 4\n",
      " processing epoch 5\n",
      " processing epoch 6\n",
      " processing epoch 7\n",
      " processing epoch 8\n",
      " processing epoch 9\n",
      " processing epoch 10\n",
      " processing epoch 11\n",
      " processing epoch 12\n",
      " processing epoch 13\n",
      " processing epoch 14\n",
      " processing epoch 15\n",
      " processing epoch 16\n",
      " processing epoch 17\n",
      " processing epoch 18\n",
      " processing epoch 19\n",
      "MSE: 0.7898\n",
      "\n",
      "Fold 3\n",
      " processing epoch 0\n",
      " processing epoch 1\n",
      " processing epoch 2\n",
      " processing epoch 3\n",
      " processing epoch 4\n",
      " processing epoch 5\n",
      " processing epoch 6\n",
      " processing epoch 7\n",
      " processing epoch 8\n",
      " processing epoch 9\n",
      " processing epoch 10\n",
      " processing epoch 11\n",
      " processing epoch 12\n",
      " processing epoch 13\n",
      " processing epoch 14\n",
      " processing epoch 15\n",
      " processing epoch 16\n",
      " processing epoch 17\n",
      " processing epoch 18\n",
      " processing epoch 19\n",
      "MSE: 0.7915\n"
     ]
    }
   ],
   "source": [
    "svd = SVDpp(n_factors=10, n_epochs=20, verbose=True)\n",
    "kf = KFold(n_splits=3)\n",
    "for k, (trainSet, validSet) in enumerate(kf.split(data)):\n",
    "    print(f\"\\nFold {k+1}\")\n",
    "    svd.fit(trainSet)\n",
    "    accuracy.mse(svd.test(validSet))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train the Entire Dataset and Save Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " processing epoch 0\n",
      " processing epoch 1\n",
      " processing epoch 2\n",
      " processing epoch 3\n",
      " processing epoch 4\n",
      " processing epoch 5\n",
      " processing epoch 6\n",
      " processing epoch 7\n",
      " processing epoch 8\n",
      " processing epoch 9\n",
      " processing epoch 10\n",
      " processing epoch 11\n",
      " processing epoch 12\n",
      " processing epoch 13\n",
      " processing epoch 14\n",
      " processing epoch 15\n",
      " processing epoch 16\n",
      " processing epoch 17\n",
      " processing epoch 18\n",
      " processing epoch 19\n"
     ]
    }
   ],
   "source": [
    "trainSet = data.build_full_trainset()\n",
    "svd.fit(trainSet)\n",
    "\n",
    "with open(\"hybrid/svdpp.joblib\", \"wb\") as f:\n",
    "    joblib.dump(svd, f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Link `tmdbId` with `movieId`\n",
    "Ratings trained using `movieId` as input features. However, the similarity matrix is created by the order of the metadata dataframe. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>genres</th>\n",
       "      <th>title</th>\n",
       "      <th>cast</th>\n",
       "      <th>keywords</th>\n",
       "      <th>director</th>\n",
       "      <th>desc</th>\n",
       "      <th>movieId</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>862</td>\n",
       "      <td>['animation', 'comedy', 'family']</td>\n",
       "      <td>Toy Story</td>\n",
       "      <td>['tomhanks', 'timallen', 'donrickles']</td>\n",
       "      <td>['jealousi', 'toy', 'boy', 'friendship', 'frie...</td>\n",
       "      <td>['johnlasseter']</td>\n",
       "      <td>animation comedy family tomhanks timallen donr...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>8844</td>\n",
       "      <td>['adventure', 'fantasy', 'family']</td>\n",
       "      <td>Jumanji</td>\n",
       "      <td>['robinwilliams', 'jonathanhyde', 'kirstendunst']</td>\n",
       "      <td>['disappear', 'basedonchildrensbook']</td>\n",
       "      <td>['joejohnston']</td>\n",
       "      <td>adventure fantasy family robinwilliams jonatha...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>15602</td>\n",
       "      <td>['romance', 'comedy']</td>\n",
       "      <td>Grumpier Old Men</td>\n",
       "      <td>['waltermatthau', 'jacklemmon', 'annmargret']</td>\n",
       "      <td>['fish', 'bestfriend', 'duringcreditssting']</td>\n",
       "      <td>['howarddeutch']</td>\n",
       "      <td>romance comedy waltermatthau jacklemmon annmar...</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>31357</td>\n",
       "      <td>['comedy', 'drama', 'romance']</td>\n",
       "      <td>Waiting to Exhale</td>\n",
       "      <td>['whitneyhouston', 'angelabassett', 'lorettade...</td>\n",
       "      <td>['basedonnovel', 'interracialrelationship', 's...</td>\n",
       "      <td>['forestwhitaker']</td>\n",
       "      <td>comedy drama romance whitneyhouston angelabass...</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>11862</td>\n",
       "      <td>['comedy']</td>\n",
       "      <td>Father of the Bride Part II</td>\n",
       "      <td>['stevemartin', 'dianekeaton', 'martinshort']</td>\n",
       "      <td>['babi', 'midlifecrisi', 'confid', 'age', 'dau...</td>\n",
       "      <td>['charlesshyer']</td>\n",
       "      <td>comedy stevemartin dianekeaton martinshort bab...</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      id                              genres                        title  \\\n",
       "0    862   ['animation', 'comedy', 'family']                    Toy Story   \n",
       "1   8844  ['adventure', 'fantasy', 'family']                      Jumanji   \n",
       "2  15602               ['romance', 'comedy']             Grumpier Old Men   \n",
       "3  31357      ['comedy', 'drama', 'romance']            Waiting to Exhale   \n",
       "4  11862                          ['comedy']  Father of the Bride Part II   \n",
       "\n",
       "                                                cast  \\\n",
       "0             ['tomhanks', 'timallen', 'donrickles']   \n",
       "1  ['robinwilliams', 'jonathanhyde', 'kirstendunst']   \n",
       "2      ['waltermatthau', 'jacklemmon', 'annmargret']   \n",
       "3  ['whitneyhouston', 'angelabassett', 'lorettade...   \n",
       "4      ['stevemartin', 'dianekeaton', 'martinshort']   \n",
       "\n",
       "                                            keywords            director  \\\n",
       "0  ['jealousi', 'toy', 'boy', 'friendship', 'frie...    ['johnlasseter']   \n",
       "1              ['disappear', 'basedonchildrensbook']     ['joejohnston']   \n",
       "2       ['fish', 'bestfriend', 'duringcreditssting']    ['howarddeutch']   \n",
       "3  ['basedonnovel', 'interracialrelationship', 's...  ['forestwhitaker']   \n",
       "4  ['babi', 'midlifecrisi', 'confid', 'age', 'dau...    ['charlesshyer']   \n",
       "\n",
       "                                                desc  movieId  \n",
       "0  animation comedy family tomhanks timallen donr...        1  \n",
       "1  adventure fantasy family robinwilliams jonatha...        2  \n",
       "2  romance comedy waltermatthau jacklemmon annmar...        3  \n",
       "3  comedy drama romance whitneyhouston angelabass...        4  \n",
       "4  comedy stevemartin dianekeaton martinshort bab...        5  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "links = pd.read_csv(\"../data/movies/links_small.csv\")[[\"movieId\", \"tmdbId\"]]\n",
    "links = links[links[\"tmdbId\"].notnull()]\n",
    "links[\"movieId\"] = links[\"movieId\"].astype(\"int\")\n",
    "links[\"tmdbId\"] = links[\"tmdbId\"].astype(\"int\")\n",
    "\n",
    "# Rename the columns\n",
    "links.columns = [\"movieId\", \"id\"]\n",
    "\n",
    "# Merge using `id`, so we have integrated `movieId` column to the metadata\n",
    "md = md.merge(links[[\"movieId\", \"id\"]], on=\"id\")\n",
    "md.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Make Recommendations\n",
    "First, we find the most similar movies regarding to the given `title`. Based on these filtered movies and the particular `userId`, we make ranking predictions. Finally, the rankings are descendingly orderd and the topk are chosen. For `userId=1` and `userId=100`, we have different rankings for them. We did custom recommendations!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getHybridTopkRecommendations(userId, title, metadataDf, similarities, simTopk=25, topk=5):\n",
    "    idx = metadataDf.index[metadataDf[\"title\"] == title].tolist()\n",
    "    if len(idx) == 0:\n",
    "        raise ValueError(\"Title not found!\")\n",
    "    # Choose 1st item and its similarity arr\n",
    "    idx = idx[0]\n",
    "    sim = similarities[idx]\n",
    "    # Set similarity of the given title to the minimum\n",
    "    sim[idx] = sim.min()\n",
    "    # Desc sort \n",
    "    indices = np.argpartition(-sim, 1+simTopk)[1:1+simTopk]\n",
    "    movies = metadataDf.iloc[indices, :][[\"id\", \"title\", \"movieId\"]]\n",
    "    # Predict ratings and desc sort\n",
    "    movies[\"est_rating\"] = movies[\"movieId\"].apply(lambda x: svd.predict(userId, x).est)\n",
    "    movies = movies.sort_values(\"est_rating\", ascending=False)\n",
    "    movies = movies.iloc[:topk, :]\n",
    "    movieRatingPair = dict(zip(movies[\"title\"], movies[\"est_rating\"]))\n",
    "    \n",
    "    print(f\"\\nUser {userId} liked {title}. He/She may like...\")\n",
    "    for i, (title, rating) in enumerate(movieRatingPair.items()):\n",
    "        print(f\"Top{i+1}: {title} {rating:.3f}\")\n",
    "    return "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "User 1 liked The Shawshank Redemption. He/She may like...\n",
      "Top1: The Godfather 3.214\n",
      "Top2: In Bruges 3.084\n",
      "Top3: Once Upon a Time in America 3.037\n",
      "Top4: Witness 3.023\n",
      "Top5: Murder in the First 3.014\n",
      "\n",
      "User 1 liked The Truman Show. He/She may like...\n",
      "Top1: Naked 3.080\n",
      "Top2: The Decline of the American Empire 2.820\n",
      "Top3: Swimming to Cambodia 2.765\n",
      "Top4: The Man in the White Suit 2.750\n",
      "Top5: The Lobster 2.723\n",
      "\n",
      "User 1 liked The Godfather. He/She may like...\n",
      "Top1: The Shawshank Redemption 3.390\n",
      "Top2: On the Waterfront 3.282\n",
      "Top3: In Bruges 3.084\n",
      "Top4: Murder in the First 3.014\n",
      "Top5: Rounders 2.942\n",
      "\n",
      "User 100 liked The Shawshank Redemption. He/She may like...\n",
      "Top1: The Godfather 4.320\n",
      "Top2: Once Upon a Time in America 4.027\n",
      "Top3: In Bruges 4.022\n",
      "Top4: Witness 3.878\n",
      "Top5: Murder in the First 3.808\n",
      "\n",
      "User 100 liked The Truman Show. He/She may like...\n",
      "Top1: Naked 3.935\n",
      "Top2: The Decline of the American Empire 3.696\n",
      "Top3: Swimming to Cambodia 3.677\n",
      "Top4: The Man in the White Suit 3.592\n",
      "Top5: Monster in a Box 3.530\n",
      "\n",
      "User 100 liked The Godfather. He/She may like...\n",
      "Top1: The Shawshank Redemption 4.377\n",
      "Top2: On the Waterfront 4.221\n",
      "Top3: In Bruges 4.022\n",
      "Top4: Donnie Brasco 3.834\n",
      "Top5: Murder in the First 3.808\n"
     ]
    }
   ],
   "source": [
    "userId=1\n",
    "getHybridTopkRecommendations(userId, \"The Shawshank Redemption\", md, dists)\n",
    "getHybridTopkRecommendations(userId, \"The Truman Show\", md, dists)\n",
    "getHybridTopkRecommendations(userId, \"The Godfather\", md, dists)\n",
    "\n",
    "userId=100\n",
    "getHybridTopkRecommendations(userId, \"The Shawshank Redemption\", md, dists)\n",
    "getHybridTopkRecommendations(userId, \"The Truman Show\", md, dists)\n",
    "getHybridTopkRecommendations(userId, \"The Godfather\", md, dists)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}