generating_dataset.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 90,
   "id": "e3e421ca-20dc-4a42-9b35-9745ea5aaa48",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "# 필요한 Surprise 알고리즘 불러오기\n",
    "from surprise import BaselineOnly \n",
    "from surprise import KNNWithMeans, KNNBaseline, KNNBasic\n",
    "from surprise import SVD\n",
    "from surprise import SVDpp\n",
    "from surprise import NMF\n",
    "from surprise import Dataset\n",
    "from surprise import accuracy\n",
    "from surprise import Reader\n",
    "from surprise.model_selection import cross_validate\n",
    "from surprise.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "id": "ccd74c0c-a319-454d-b0e4-fbcc93b5f983",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['data.zip',\n",
       " 'final_rating(sentiment,emotion).csv',\n",
       " 'ml-100k.zip',\n",
       " 'movie_info.csv',\n",
       " 'rotten_rating(review_20_user).csv',\n",
       " 'rotten_rating(review_30_user).csv',\n",
       " 'rotten_tomatoes_movies.csv',\n",
       " 'rotten_user_table.csv',\n",
       " 'save_surprise.zip',\n",
       " 's_testset.csv',\n",
       " 's_testset_removed.csv',\n",
       " 's_trainset.csv',\n",
       " 'testset_filtered.csv',\n",
       " 'testset_removed.csv',\n",
       " 'trainset_filtered.csv',\n",
       " 'user_info.csv']"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path = './../data_bigcomp/'\n",
    "os.listdir(path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "89d2c5e5-10a6-477e-be28-c4a8c04b4334",
   "metadata": {},
   "outputs": [],
   "source": [
    "# csv 파일에서 불러오기\n",
    "# r_cols = ['user_id', 'movie_id', 'rating', 'timestamp']\n",
    "ratings_df = pd.read_csv(path + 'final_rating(sentiment,emotion).csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "id": "359dba70-504f-4d09-9306-af6c133e648d",
   "metadata": {},
   "outputs": [],
   "source": [
    "ratings_df = ratings_df[['user_id', 'movie_id', 'review_score' , 'sentiment', 'emotion', 'review_date', 'review_content']]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3ef55322-f68f-48e4-9206-a12876324e7d",
   "metadata": {},
   "source": [
    "- 0.5 level 스케일"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "id": "dfe20348-64ae-4ea7-9975-27202c3e7dd3",
   "metadata": {},
   "outputs": [],
   "source": [
    "ratings_df['origin_rating_0.5'] = ratings_df.review_score * 4.5 + 0.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "id": "26a328e4-c3c5-407b-8b97-fa2a0489d412",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 소수점 0.5 단위로 반내림\n",
    "scaled_rating = list()\n",
    "\n",
    "for val in ratings_df['origin_rating_0.5']:\n",
    "    remainder = 0\n",
    "    if val % 0.5 != 0.0:\n",
    "        remainder = val % 0.5\n",
    "    scaled_rating.append(val - remainder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "id": "31f16c1e-a44e-4177-9095-a9edc6172379",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0.5, 1.0, 1.5, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5.0}"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set(scaled_rating)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "id": "4fd91f86-3f6c-406a-9883-df9ea59e4334",
   "metadata": {},
   "outputs": [],
   "source": [
    "ratings_df['rating_0.5'] = scaled_rating"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "02090934-2820-4fed-acc8-1167d862684a",
   "metadata": {},
   "source": [
    "- 1 level 스케일"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "id": "7428cb67-bcfd-47ad-be60-c995cdcb8fdf",
   "metadata": {},
   "outputs": [],
   "source": [
    "ratings_df['origin_rating_1'] = ratings_df.review_score * 4 + 1.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "id": "78a771c3-7e39-42ba-9097-fff59c532c08",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0         3.8\n",
       "1         2.0\n",
       "2         3.8\n",
       "3         3.8\n",
       "4         3.4\n",
       "         ... \n",
       "751124    5.0\n",
       "751125    3.8\n",
       "751126    3.8\n",
       "751127    3.8\n",
       "751128    3.0\n",
       "Name: origin_rating_1, Length: 751129, dtype: float64"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ratings_df['origin_rating_1']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "id": "ab687b67-5ab8-42be-a1cb-7c7dabb2c083",
   "metadata": {},
   "outputs": [],
   "source": [
    "scaled_rating = list()\n",
    "\n",
    "for val in ratings_df.origin_rating_1:\n",
    "    scaled_rating.append(round(val))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "id": "26ce6f54-8869-4d79-ad10-d4900d08c076",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{1, 2, 3, 4, 5}"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set(scaled_rating)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "id": "3954d96a-f148-42bb-a25f-d0d91360c6ad",
   "metadata": {},
   "outputs": [],
   "source": [
    "ratings_df['rating_1'] = scaled_rating"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "id": "e246d32e-795e-4074-b798-f013e5706639",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>movie_id</th>\n",
       "      <th>review_score</th>\n",
       "      <th>sentiment</th>\n",
       "      <th>emotion</th>\n",
       "      <th>review_date</th>\n",
       "      <th>review_content</th>\n",
       "      <th>origin_rating_0.5</th>\n",
       "      <th>rating_0.5</th>\n",
       "      <th>origin_rating_1</th>\n",
       "      <th>rating_1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>943</td>\n",
       "      <td>0</td>\n",
       "      <td>0.70</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2010-02-09</td>\n",
       "      <td>Whether audiences will get behind The Lightnin...</td>\n",
       "      <td>3.650</td>\n",
       "      <td>3.5</td>\n",
       "      <td>3.8</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7242</td>\n",
       "      <td>0</td>\n",
       "      <td>0.25</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2010-02-10</td>\n",
       "      <td>Harry Potter knockoffs don't come more transpa...</td>\n",
       "      <td>1.625</td>\n",
       "      <td>1.5</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1046</td>\n",
       "      <td>0</td>\n",
       "      <td>0.70</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>2010-02-10</td>\n",
       "      <td>Percy Jackson isn't a great movie, but it's a ...</td>\n",
       "      <td>3.650</td>\n",
       "      <td>3.5</td>\n",
       "      <td>3.8</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4895</td>\n",
       "      <td>0</td>\n",
       "      <td>0.70</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>2010-02-10</td>\n",
       "      <td>Fun, brisk and imaginative</td>\n",
       "      <td>3.650</td>\n",
       "      <td>3.5</td>\n",
       "      <td>3.8</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4517</td>\n",
       "      <td>0</td>\n",
       "      <td>0.60</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2010-02-10</td>\n",
       "      <td>Crammed with dragons, set-destroying fights an...</td>\n",
       "      <td>3.200</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.4</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   user_id  movie_id  review_score  sentiment  emotion review_date  \\\n",
       "0      943         0          0.70          2        2  2010-02-09   \n",
       "1     7242         0          0.25          0        2  2010-02-10   \n",
       "2     1046         0          0.70          3        2  2010-02-10   \n",
       "3     4895         0          0.70          4        2  2010-02-10   \n",
       "4     4517         0          0.60          1        2  2010-02-10   \n",
       "\n",
       "                                      review_content  origin_rating_0.5  \\\n",
       "0  Whether audiences will get behind The Lightnin...              3.650   \n",
       "1  Harry Potter knockoffs don't come more transpa...              1.625   \n",
       "2  Percy Jackson isn't a great movie, but it's a ...              3.650   \n",
       "3                         Fun, brisk and imaginative              3.650   \n",
       "4  Crammed with dragons, set-destroying fights an...              3.200   \n",
       "\n",
       "   rating_0.5  origin_rating_1  rating_1  \n",
       "0         3.5              3.8         4  \n",
       "1         1.5              2.0         2  \n",
       "2         3.5              3.8         4  \n",
       "3         3.5              3.8         4  \n",
       "4         3.0              3.4         3  "
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ratings_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7c19341e-a165-4e2e-9a46-121d01c3b557",
   "metadata": {},
   "source": [
    "### 1. 30개 이상의 리뷰 남긴 User"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "id": "afab4ad2-d99c-44c3-a4ba-95a5182665dc",
   "metadata": {},
   "outputs": [],
   "source": [
    "group = ratings_df.groupby('user_id')\n",
    "group_user_id = group['movie_id'].count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "id": "215e312f-b7ca-4072-ad41-75b7ca5d3b82",
   "metadata": {},
   "outputs": [],
   "source": [
    "def cal_rate(num, total, limit, condition):\n",
    "    if condition == 'low':\n",
    "        print(f\"{limit}개 미만: {sum(num < limit)}, {round((sum(num < limit)/total),2)*100}%\")\n",
    "        return sum(num >= limit)\n",
    "    \n",
    "    if condition == 'high':\n",
    "        print(f\"{limit}개 이상: {sum(num >= limit)}, {round((sum(num > limit)/total),2)*100}%\")\n",
    "        return sum(num >= limit)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "id": "3b0c0446-f60f-479a-95f4-c248832396a0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "30개 이상: 2490, 25.0%\n"
     ]
    }
   ],
   "source": [
    "num = group_user_id.sort_values()\n",
    "total = len(group_user_id)\n",
    "\n",
    "condition = 'high'\n",
    "sample_num = cal_rate(num, total, 30, condition)\n",
    "# sample_num = cal_rate(num, total, 100, condition)\n",
    "# sample_num = cal_rate(num, total, 500, condition)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "id": "ca348b29-4f8f-432c-9c38-c19142b3020a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "user_id\n",
       "2600    6486\n",
       "8148    6356\n",
       "3265    5300\n",
       "2495    5212\n",
       "3926    4548\n",
       "Name: movie_id, dtype: int64"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "group_user_id.sort_values(ascending=False)[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "id": "607f1568-80f1-4916-b164-e14e1fd1f569",
   "metadata": {},
   "outputs": [],
   "source": [
    "sample = group_user_id.sort_values(ascending=False)[:sample_num]\n",
    "total = len(group_user_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "id": "46713a42-08eb-4961-a7b9-7f342723fdff",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2490"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(sample)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "id": "6fe5ff06-d47c-4a3f-a3bb-45119707d7c0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Int64Index([2600, 8148, 3265, 2495, 3926, 5446, 5716, 7603, 7917, 1738,\n",
       "            ...\n",
       "            6998,  588, 3584, 4627, 6994, 2446, 5048,   70, 6736, 2715],\n",
       "           dtype='int64', name='user_id', length=2490)"
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample.index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "id": "29179ead-f447-4780-852a-f62e4e360fb7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 특정 인덱스에 해당하는 행(리뷰 100개 이상 남긴 유저)만 필터링\n",
    "df = ratings_df[ratings_df['user_id'].isin(list(sample.index))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "id": "9137666d-2b3b-4003-ad48-cfcfce53cfa4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(708917, 11)"
      ]
     },
     "execution_count": 112,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "id": "af844f3d-9f03-472f-bcab-8e93074eefe7",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv(path+'rotten_rating(review_30_user).csv', index=False)\n",
    "# df.to_csv(path+'rotten_rating(review_100_user).csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dec17c34-dd88-417a-a82a-d0c7184cb03c",
   "metadata": {},
   "source": [
    "### 2. 평점 파일 로드하기(리뷰 30 이상)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "id": "739accbf-713f-43e5-a7a8-387e1af273ab",
   "metadata": {},
   "outputs": [],
   "source": [
    "ratings_df = pd.read_csv(path + 'rotten_rating(review_30_user).csv')\n",
    "# ratings_df = pd.read_csv(path + 'rotten_rating(review_100_user).csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "id": "a444b27a-d02f-4458-82d1-74cc680fd472",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>movie_id</th>\n",
       "      <th>review_score</th>\n",
       "      <th>sentiment</th>\n",
       "      <th>emotion</th>\n",
       "      <th>review_date</th>\n",
       "      <th>review_content</th>\n",
       "      <th>origin_rating_0.5</th>\n",
       "      <th>rating_0.5</th>\n",
       "      <th>origin_rating_1</th>\n",
       "      <th>rating_1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7242</td>\n",
       "      <td>0</td>\n",
       "      <td>0.25</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2010-02-10</td>\n",
       "      <td>Harry Potter knockoffs don't come more transpa...</td>\n",
       "      <td>1.625</td>\n",
       "      <td>1.5</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1046</td>\n",
       "      <td>0</td>\n",
       "      <td>0.70</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>2010-02-10</td>\n",
       "      <td>Percy Jackson isn't a great movie, but it's a ...</td>\n",
       "      <td>3.650</td>\n",
       "      <td>3.5</td>\n",
       "      <td>3.8</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4895</td>\n",
       "      <td>0</td>\n",
       "      <td>0.70</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>2010-02-10</td>\n",
       "      <td>Fun, brisk and imaginative</td>\n",
       "      <td>3.650</td>\n",
       "      <td>3.5</td>\n",
       "      <td>3.8</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4517</td>\n",
       "      <td>0</td>\n",
       "      <td>0.60</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2010-02-10</td>\n",
       "      <td>Crammed with dragons, set-destroying fights an...</td>\n",
       "      <td>3.200</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.4</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6226</td>\n",
       "      <td>0</td>\n",
       "      <td>0.80</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>2010-02-10</td>\n",
       "      <td>This action-packed fantasy adventure, based on...</td>\n",
       "      <td>4.100</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.2</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   user_id  movie_id  review_score  sentiment  emotion review_date  \\\n",
       "0     7242         0          0.25          0        2  2010-02-10   \n",
       "1     1046         0          0.70          3        2  2010-02-10   \n",
       "2     4895         0          0.70          4        2  2010-02-10   \n",
       "3     4517         0          0.60          1        2  2010-02-10   \n",
       "4     6226         0          0.80          4        2  2010-02-10   \n",
       "\n",
       "                                      review_content  origin_rating_0.5  \\\n",
       "0  Harry Potter knockoffs don't come more transpa...              1.625   \n",
       "1  Percy Jackson isn't a great movie, but it's a ...              3.650   \n",
       "2                         Fun, brisk and imaginative              3.650   \n",
       "3  Crammed with dragons, set-destroying fights an...              3.200   \n",
       "4  This action-packed fantasy adventure, based on...              4.100   \n",
       "\n",
       "   rating_0.5  origin_rating_1  rating_1  \n",
       "0         1.5              2.0         2  \n",
       "1         3.5              3.8         4  \n",
       "2         3.5              3.8         4  \n",
       "3         3.0              3.4         3  \n",
       "4         4.0              4.2         4  "
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ratings_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "id": "29595751-2adf-409f-b273-decdcef5a797",
   "metadata": {},
   "outputs": [],
   "source": [
    "ratings_df['review_date'] = pd.to_datetime(ratings_df.review_date)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "id": "7bb7ba4f-f3e7-43f2-9f13-52182d42b0fe",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "user_id              0\n",
       "movie_id             0\n",
       "review_score         0\n",
       "sentiment            0\n",
       "emotion              0\n",
       "review_date          0\n",
       "review_content       0\n",
       "origin_rating_0.5    0\n",
       "rating_0.5           0\n",
       "origin_rating_1      0\n",
       "rating_1             0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 117,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ratings_df.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "id": "0ff180f2-dfc0-48e8-b706-bfc4f5b355ed",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 날짜 별로 정렬\n",
    "ratings_df.sort_values(by='review_date', inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "id": "f67b4571-fd21-46d2-a55d-fbfb77464a9e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>movie_id</th>\n",
       "      <th>review_score</th>\n",
       "      <th>sentiment</th>\n",
       "      <th>emotion</th>\n",
       "      <th>review_date</th>\n",
       "      <th>review_content</th>\n",
       "      <th>origin_rating_0.5</th>\n",
       "      <th>rating_0.5</th>\n",
       "      <th>origin_rating_1</th>\n",
       "      <th>rating_1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>253595</th>\n",
       "      <td>9296</td>\n",
       "      <td>7594</td>\n",
       "      <td>0.600</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>1800-01-01</td>\n",
       "      <td>It's exciting to see a British horror film wit...</td>\n",
       "      <td>3.2000</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.4</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72106</th>\n",
       "      <td>7403</td>\n",
       "      <td>2494</td>\n",
       "      <td>1.000</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>1800-01-01</td>\n",
       "      <td>A grimly seductive end-of-the-world thriller, ...</td>\n",
       "      <td>5.0000</td>\n",
       "      <td>5.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>112724</th>\n",
       "      <td>9296</td>\n",
       "      <td>3644</td>\n",
       "      <td>0.800</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1800-01-01</td>\n",
       "      <td>There aren't many British horror films you cou...</td>\n",
       "      <td>4.1000</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.2</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>299580</th>\n",
       "      <td>7403</td>\n",
       "      <td>8800</td>\n",
       "      <td>0.700</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>1800-01-01</td>\n",
       "      <td>Coming out from behind Spike Lee's camera, Ern...</td>\n",
       "      <td>3.6500</td>\n",
       "      <td>3.5</td>\n",
       "      <td>3.8</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>245700</th>\n",
       "      <td>7629</td>\n",
       "      <td>7374</td>\n",
       "      <td>0.625</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>1800-01-01</td>\n",
       "      <td>A comedy western loaded with chuckles that onl...</td>\n",
       "      <td>3.3125</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.5</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        user_id  movie_id  review_score  sentiment  emotion review_date  \\\n",
       "253595     9296      7594         0.600          4        2  1800-01-01   \n",
       "72106      7403      2494         1.000          3        4  1800-01-01   \n",
       "112724     9296      3644         0.800          1        2  1800-01-01   \n",
       "299580     7403      8800         0.700          4        2  1800-01-01   \n",
       "245700     7629      7374         0.625          3        4  1800-01-01   \n",
       "\n",
       "                                           review_content  origin_rating_0.5  \\\n",
       "253595  It's exciting to see a British horror film wit...             3.2000   \n",
       "72106   A grimly seductive end-of-the-world thriller, ...             5.0000   \n",
       "112724  There aren't many British horror films you cou...             4.1000   \n",
       "299580  Coming out from behind Spike Lee's camera, Ern...             3.6500   \n",
       "245700  A comedy western loaded with chuckles that onl...             3.3125   \n",
       "\n",
       "        rating_0.5  origin_rating_1  rating_1  \n",
       "253595         3.0              3.4         3  \n",
       "72106          5.0              5.0         5  \n",
       "112724         4.0              4.2         4  \n",
       "299580         3.5              3.8         4  \n",
       "245700         3.0              3.5         4  "
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ratings_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "id": "e72ec96c-a392-4e83-8cb6-f14a5f998dd6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# from sklearn.model_selection import train_test_split\n",
    "# trainset, testset, _, _ = train_test_split(ratings_df, ratings_df, test_size=0.3, random_state=24)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "10cc6de5-8311-4f62-bd59-97ebac87a855",
   "metadata": {},
   "source": [
    "### 3. dataset 분할시 cold-start problem case 분리"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "id": "cfb4ba13-839f-4055-bc55-835bf38313ed",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(708917, 11)"
      ]
     },
     "execution_count": 121,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ratings_df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "05aaca7e-fdc6-474f-9f6b-5a94d3c69646",
   "metadata": {},
   "source": [
    "- timestamp가 2010년도 이후만 조회"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "id": "6825c15a-3fbc-4171-8053-6a9529d060b5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "253595    False\n",
       "72106     False\n",
       "112724    False\n",
       "299580    False\n",
       "245700    False\n",
       "          ...  \n",
       "584382     True\n",
       "702576     True\n",
       "676277     True\n",
       "676021     True\n",
       "700260     True\n",
       "Name: review_date, Length: 708917, dtype: bool"
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ratings_df.review_date > '1990-01-01'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "id": "a25cabfb-3d62-4974-bd4d-d0e5d3a34984",
   "metadata": {},
   "outputs": [],
   "source": [
    "# sum((ratings_df.review_date < '2011-01-01') & (ratings_df.review_date > '2000-01-01'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "id": "d354297e-1b6f-45dd-a56e-696acf92d9a9",
   "metadata": {},
   "outputs": [],
   "source": [
    "new_ratings_df = ratings_df[(ratings_df.review_date < '2018-01-01')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "id": "0e2a1354-b7a2-4f3b-9ca1-7e6ab714c417",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "594402"
      ]
     },
     "execution_count": 125,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(new_ratings_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "id": "54e906d1-e298-42bb-ac3e-82aec5dbb8af",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "416082\n"
     ]
    }
   ],
   "source": [
    "_test_ratio = 0.70 # test set 비율\n",
    "num_test = int(np.ceil(new_ratings_df.shape[0] * _test_ratio))\n",
    "print(num_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "id": "f29b318c-9672-4335-9edf-30486152b277",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "trainset 개수: 178320\n",
      "testset 개수: 416082\n"
     ]
    }
   ],
   "source": [
    "trainset = new_ratings_df.iloc[:-num_test]\n",
    "testset  = new_ratings_df.iloc[-num_test:]\n",
    "\n",
    "print(f\"trainset 개수: {len(trainset)}\")\n",
    "print(f\"testset 개수: {len(testset)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "id": "473aadc1-8bc3-4c3d-bd18-7fe9d59c675b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "trainset user: 855\n",
      "trainset movie: 7844\n",
      "testset user: 2015\n",
      "testset movie: 14090\n"
     ]
    }
   ],
   "source": [
    "trainset_user_id = set(trainset.user_id)\n",
    "trainset_movie_id = set(trainset.movie_id)\n",
    "print(f\"trainset user: {len(trainset_user_id)}\")\n",
    "print(f\"trainset movie: {len(trainset_movie_id)}\")\n",
    "\n",
    "testset_user_id = set(testset.user_id)\n",
    "testset_movie_id = set(testset.movie_id)\n",
    "print(f\"testset user: {len(testset_user_id)}\")\n",
    "print(f\"testset movie: {len(testset_movie_id)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "id": "1b9ac20e-c4ff-49b5-a65b-ec0cd644b2b0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "교집합 user_id: 708\n",
      "교집합 movie_id: 6201\n",
      "차집합 user_id: 1307\n",
      "차집합 movie_id: 7889\n"
     ]
    }
   ],
   "source": [
    "intersection_user_id = trainset_user_id.intersection(testset_user_id)\n",
    "intersection_movie_id = trainset_movie_id.intersection(testset_movie_id)\n",
    "print(f\"교집합 user_id: {len(intersection_user_id)}\")\n",
    "print(f\"교집합 movie_id: {len(intersection_movie_id)}\")\n",
    "\n",
    "difference_user_id = testset_user_id.difference(trainset_user_id)\n",
    "difference_movie_id = testset_movie_id.difference(trainset_movie_id)\n",
    "print(f\"차집합 user_id: {len(difference_user_id)}\") # testset에만 있는 user_id\n",
    "print(f\"차집합 movie_id: {len(difference_movie_id)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f754da3a-c710-4740-aff2-6bbfeedf6159",
   "metadata": {},
   "source": [
    "#### 3-1. testset에서 교집합 user_id, movie_id 데이터만 추출"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "id": "888df1c2-fddb-48ce-836c-1eb8789881de",
   "metadata": {},
   "outputs": [],
   "source": [
    "condition_inter_user  = testset['user_id'].isin(intersection_user_id)\n",
    "condition_inter_movie = testset['movie_id'].isin(intersection_movie_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "id": "bdfab73f-2f7b-48a0-8926-ac3b563f34ce",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "26851"
      ]
     },
     "execution_count": 131,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum(condition_inter_user & condition_inter_movie)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "id": "7df7f6b7-c7cf-45af-b978-733ccbef502b",
   "metadata": {},
   "outputs": [],
   "source": [
    "filtered_testset = testset[condition_inter_user & condition_inter_movie].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "id": "fe07bab4-c3f1-4b81-be28-035563aa968f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>movie_id</th>\n",
       "      <th>review_score</th>\n",
       "      <th>sentiment</th>\n",
       "      <th>emotion</th>\n",
       "      <th>review_date</th>\n",
       "      <th>review_content</th>\n",
       "      <th>origin_rating_0.5</th>\n",
       "      <th>rating_0.5</th>\n",
       "      <th>origin_rating_1</th>\n",
       "      <th>rating_1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>392623</th>\n",
       "      <td>5127</td>\n",
       "      <td>11393</td>\n",
       "      <td>0.800</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>2006-06-21</td>\n",
       "      <td>Thoroughly satisfying... manages to tackle its...</td>\n",
       "      <td>4.1000</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.2</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>415285</th>\n",
       "      <td>4327</td>\n",
       "      <td>11993</td>\n",
       "      <td>0.700</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>2006-06-21</td>\n",
       "      <td>I wonder if anyone will consider Keillor for a...</td>\n",
       "      <td>3.6500</td>\n",
       "      <td>3.5</td>\n",
       "      <td>3.8</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>257499</th>\n",
       "      <td>7403</td>\n",
       "      <td>7725</td>\n",
       "      <td>0.700</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>2006-06-21</td>\n",
       "      <td>The Hidden Blade is tranquil, touching, and, i...</td>\n",
       "      <td>3.6500</td>\n",
       "      <td>3.5</td>\n",
       "      <td>3.8</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>415284</th>\n",
       "      <td>4327</td>\n",
       "      <td>11993</td>\n",
       "      <td>0.700</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>2006-06-21</td>\n",
       "      <td>I wonder if anyone will consider Keillor for a...</td>\n",
       "      <td>3.6500</td>\n",
       "      <td>3.5</td>\n",
       "      <td>3.8</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>454947</th>\n",
       "      <td>2858</td>\n",
       "      <td>12595</td>\n",
       "      <td>0.850</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>2006-06-21</td>\n",
       "      <td>Up there with United 93 as one of the most pow...</td>\n",
       "      <td>4.3250</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.4</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>178131</th>\n",
       "      <td>2463</td>\n",
       "      <td>5465</td>\n",
       "      <td>0.875</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2017-12-02</td>\n",
       "      <td>What gives Zombie the nerve, the sheer audacit...</td>\n",
       "      <td>4.4375</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.5</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>371926</th>\n",
       "      <td>4367</td>\n",
       "      <td>10733</td>\n",
       "      <td>0.800</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>2017-12-06</td>\n",
       "      <td>No party is as joyous as a Muppet celebration,...</td>\n",
       "      <td>4.1000</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.2</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>334518</th>\n",
       "      <td>8391</td>\n",
       "      <td>9771</td>\n",
       "      <td>0.700</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>2017-12-08</td>\n",
       "      <td>As I've said, it's not entirely rinsed free of...</td>\n",
       "      <td>3.6500</td>\n",
       "      <td>3.5</td>\n",
       "      <td>3.8</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19265</th>\n",
       "      <td>2463</td>\n",
       "      <td>1052</td>\n",
       "      <td>0.750</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>2017-12-25</td>\n",
       "      <td>The movie creates an atmosphere that is tense ...</td>\n",
       "      <td>3.8750</td>\n",
       "      <td>3.5</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18754</th>\n",
       "      <td>2562</td>\n",
       "      <td>1015</td>\n",
       "      <td>0.400</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2017-12-30</td>\n",
       "      <td>If it seems implausible that a young woman on ...</td>\n",
       "      <td>2.3000</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.6</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>26851 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        user_id  movie_id  review_score  sentiment  emotion review_date  \\\n",
       "392623     5127     11393         0.800          4        2  2006-06-21   \n",
       "415285     4327     11993         0.700          4        2  2006-06-21   \n",
       "257499     7403      7725         0.700          4        2  2006-06-21   \n",
       "415284     4327     11993         0.700          4        2  2006-06-21   \n",
       "454947     2858     12595         0.850          4        2  2006-06-21   \n",
       "...         ...       ...           ...        ...      ...         ...   \n",
       "178131     2463      5465         0.875          1        0  2017-12-02   \n",
       "371926     4367     10733         0.800          3        2  2017-12-06   \n",
       "334518     8391      9771         0.700          3        2  2017-12-08   \n",
       "19265      2463      1052         0.750          4        0  2017-12-25   \n",
       "18754      2562      1015         0.400          1        2  2017-12-30   \n",
       "\n",
       "                                           review_content  origin_rating_0.5  \\\n",
       "392623  Thoroughly satisfying... manages to tackle its...             4.1000   \n",
       "415285  I wonder if anyone will consider Keillor for a...             3.6500   \n",
       "257499  The Hidden Blade is tranquil, touching, and, i...             3.6500   \n",
       "415284  I wonder if anyone will consider Keillor for a...             3.6500   \n",
       "454947  Up there with United 93 as one of the most pow...             4.3250   \n",
       "...                                                   ...                ...   \n",
       "178131  What gives Zombie the nerve, the sheer audacit...             4.4375   \n",
       "371926  No party is as joyous as a Muppet celebration,...             4.1000   \n",
       "334518  As I've said, it's not entirely rinsed free of...             3.6500   \n",
       "19265   The movie creates an atmosphere that is tense ...             3.8750   \n",
       "18754   If it seems implausible that a young woman on ...             2.3000   \n",
       "\n",
       "        rating_0.5  origin_rating_1  rating_1  \n",
       "392623         4.0              4.2         4  \n",
       "415285         3.5              3.8         4  \n",
       "257499         3.5              3.8         4  \n",
       "415284         3.5              3.8         4  \n",
       "454947         4.0              4.4         4  \n",
       "...            ...              ...       ...  \n",
       "178131         4.0              4.5         4  \n",
       "371926         4.0              4.2         4  \n",
       "334518         3.5              3.8         4  \n",
       "19265          3.5              4.0         4  \n",
       "18754          2.0              2.6         3  \n",
       "\n",
       "[26851 rows x 11 columns]"
      ]
     },
     "execution_count": 133,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filtered_testset"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ad958e48-e6da-4c8c-9b70-d136a8ef40f4",
   "metadata": {},
   "source": [
    "#### 3-2. testset에서 차집합 user_id, movie_id 데이터만 추출 (cold-start용)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "id": "1d7c5104-da91-487c-918d-8c2aa6d371bb",
   "metadata": {},
   "outputs": [],
   "source": [
    "condition_differ_user  = testset['user_id'].isin(difference_user_id)\n",
    "condition_differ_movie = testset['movie_id'].isin(difference_movie_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "id": "e2164de8-0b37-430b-bcc0-f5dd239b13a6",
   "metadata": {},
   "outputs": [],
   "source": [
    "removed_testset = testset[condition_inter_user | condition_inter_movie].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "id": "6f3be2cc-e7d3-4cfc-af94-9d76c40419f9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>movie_id</th>\n",
       "      <th>review_score</th>\n",
       "      <th>sentiment</th>\n",
       "      <th>emotion</th>\n",
       "      <th>review_date</th>\n",
       "      <th>review_content</th>\n",
       "      <th>origin_rating_0.5</th>\n",
       "      <th>rating_0.5</th>\n",
       "      <th>origin_rating_1</th>\n",
       "      <th>rating_1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>392623</th>\n",
       "      <td>5127</td>\n",
       "      <td>11393</td>\n",
       "      <td>0.80</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>2006-06-21</td>\n",
       "      <td>Thoroughly satisfying... manages to tackle its...</td>\n",
       "      <td>4.100</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.2</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>415285</th>\n",
       "      <td>4327</td>\n",
       "      <td>11993</td>\n",
       "      <td>0.70</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>2006-06-21</td>\n",
       "      <td>I wonder if anyone will consider Keillor for a...</td>\n",
       "      <td>3.650</td>\n",
       "      <td>3.5</td>\n",
       "      <td>3.8</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>257499</th>\n",
       "      <td>7403</td>\n",
       "      <td>7725</td>\n",
       "      <td>0.70</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>2006-06-21</td>\n",
       "      <td>The Hidden Blade is tranquil, touching, and, i...</td>\n",
       "      <td>3.650</td>\n",
       "      <td>3.5</td>\n",
       "      <td>3.8</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>679890</th>\n",
       "      <td>7242</td>\n",
       "      <td>16970</td>\n",
       "      <td>0.25</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>2006-06-21</td>\n",
       "      <td>The film's racial/class issues are less develo...</td>\n",
       "      <td>1.625</td>\n",
       "      <td>1.5</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>415284</th>\n",
       "      <td>4327</td>\n",
       "      <td>11993</td>\n",
       "      <td>0.70</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>2006-06-21</td>\n",
       "      <td>I wonder if anyone will consider Keillor for a...</td>\n",
       "      <td>3.650</td>\n",
       "      <td>3.5</td>\n",
       "      <td>3.8</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>333541</th>\n",
       "      <td>2600</td>\n",
       "      <td>9743</td>\n",
       "      <td>0.50</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2017-12-31</td>\n",
       "      <td>Its insistence on being absurd throughout beca...</td>\n",
       "      <td>2.750</td>\n",
       "      <td>2.5</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>363550</th>\n",
       "      <td>8103</td>\n",
       "      <td>10513</td>\n",
       "      <td>1.00</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>2017-12-31</td>\n",
       "      <td>Hollywood provides few good roles for women, b...</td>\n",
       "      <td>5.000</td>\n",
       "      <td>5.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>155762</th>\n",
       "      <td>8103</td>\n",
       "      <td>4783</td>\n",
       "      <td>0.70</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>2017-12-31</td>\n",
       "      <td>This is a totally unbelievable movie, but it i...</td>\n",
       "      <td>3.650</td>\n",
       "      <td>3.5</td>\n",
       "      <td>3.8</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>648251</th>\n",
       "      <td>8103</td>\n",
       "      <td>16094</td>\n",
       "      <td>0.70</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>2017-12-31</td>\n",
       "      <td>This coming of age movie reminded me a bit of ...</td>\n",
       "      <td>3.650</td>\n",
       "      <td>3.5</td>\n",
       "      <td>3.8</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>278420</th>\n",
       "      <td>8103</td>\n",
       "      <td>8293</td>\n",
       "      <td>0.70</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>2017-12-31</td>\n",
       "      <td>This story has something to say about the rise...</td>\n",
       "      <td>3.650</td>\n",
       "      <td>3.5</td>\n",
       "      <td>3.8</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>236925 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        user_id  movie_id  review_score  sentiment  emotion review_date  \\\n",
       "392623     5127     11393          0.80          4        2  2006-06-21   \n",
       "415285     4327     11993          0.70          4        2  2006-06-21   \n",
       "257499     7403      7725          0.70          4        2  2006-06-21   \n",
       "679890     7242     16970          0.25          2        4  2006-06-21   \n",
       "415284     4327     11993          0.70          4        2  2006-06-21   \n",
       "...         ...       ...           ...        ...      ...         ...   \n",
       "333541     2600      9743          0.50          0        1  2017-12-31   \n",
       "363550     8103     10513          1.00          2        4  2017-12-31   \n",
       "155762     8103      4783          0.70          3        2  2017-12-31   \n",
       "648251     8103     16094          0.70          4        2  2017-12-31   \n",
       "278420     8103      8293          0.70          3        2  2017-12-31   \n",
       "\n",
       "                                           review_content  origin_rating_0.5  \\\n",
       "392623  Thoroughly satisfying... manages to tackle its...              4.100   \n",
       "415285  I wonder if anyone will consider Keillor for a...              3.650   \n",
       "257499  The Hidden Blade is tranquil, touching, and, i...              3.650   \n",
       "679890  The film's racial/class issues are less develo...              1.625   \n",
       "415284  I wonder if anyone will consider Keillor for a...              3.650   \n",
       "...                                                   ...                ...   \n",
       "333541  Its insistence on being absurd throughout beca...              2.750   \n",
       "363550  Hollywood provides few good roles for women, b...              5.000   \n",
       "155762  This is a totally unbelievable movie, but it i...              3.650   \n",
       "648251  This coming of age movie reminded me a bit of ...              3.650   \n",
       "278420  This story has something to say about the rise...              3.650   \n",
       "\n",
       "        rating_0.5  origin_rating_1  rating_1  \n",
       "392623         4.0              4.2         4  \n",
       "415285         3.5              3.8         4  \n",
       "257499         3.5              3.8         4  \n",
       "679890         1.5              2.0         2  \n",
       "415284         3.5              3.8         4  \n",
       "...            ...              ...       ...  \n",
       "333541         2.5              3.0         3  \n",
       "363550         5.0              5.0         5  \n",
       "155762         3.5              3.8         4  \n",
       "648251         3.5              3.8         4  \n",
       "278420         3.5              3.8         4  \n",
       "\n",
       "[236925 rows x 11 columns]"
      ]
     },
     "execution_count": 136,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "removed_testset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "id": "40ce8a98-cb41-4653-9b89-49de730f4e77",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.15057761327949754"
      ]
     },
     "execution_count": 137,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(filtered_testset) / len(trainset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "id": "6362d141-6cd2-4651-9153-0a95daed40e4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>movie_id</th>\n",
       "      <th>review_score</th>\n",
       "      <th>sentiment</th>\n",
       "      <th>emotion</th>\n",
       "      <th>review_date</th>\n",
       "      <th>review_content</th>\n",
       "      <th>origin_rating_0.5</th>\n",
       "      <th>rating_0.5</th>\n",
       "      <th>origin_rating_1</th>\n",
       "      <th>rating_1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>253595</th>\n",
       "      <td>9296</td>\n",
       "      <td>7594</td>\n",
       "      <td>0.600</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>1800-01-01</td>\n",
       "      <td>It's exciting to see a British horror film wit...</td>\n",
       "      <td>3.2000</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.4</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72106</th>\n",
       "      <td>7403</td>\n",
       "      <td>2494</td>\n",
       "      <td>1.000</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>1800-01-01</td>\n",
       "      <td>A grimly seductive end-of-the-world thriller, ...</td>\n",
       "      <td>5.0000</td>\n",
       "      <td>5.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>112724</th>\n",
       "      <td>9296</td>\n",
       "      <td>3644</td>\n",
       "      <td>0.800</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1800-01-01</td>\n",
       "      <td>There aren't many British horror films you cou...</td>\n",
       "      <td>4.1000</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.2</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>299580</th>\n",
       "      <td>7403</td>\n",
       "      <td>8800</td>\n",
       "      <td>0.700</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>1800-01-01</td>\n",
       "      <td>Coming out from behind Spike Lee's camera, Ern...</td>\n",
       "      <td>3.6500</td>\n",
       "      <td>3.5</td>\n",
       "      <td>3.8</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>245700</th>\n",
       "      <td>7629</td>\n",
       "      <td>7374</td>\n",
       "      <td>0.625</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>1800-01-01</td>\n",
       "      <td>A comedy western loaded with chuckles that onl...</td>\n",
       "      <td>3.3125</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.5</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>566157</th>\n",
       "      <td>8589</td>\n",
       "      <td>14185</td>\n",
       "      <td>0.700</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2006-06-21</td>\n",
       "      <td>... leaves the comic-book super-villain melodr...</td>\n",
       "      <td>3.6500</td>\n",
       "      <td>3.5</td>\n",
       "      <td>3.8</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>563361</th>\n",
       "      <td>4281</td>\n",
       "      <td>14097</td>\n",
       "      <td>0.700</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2006-06-21</td>\n",
       "      <td>Singer's work is generally strong, but Superma...</td>\n",
       "      <td>3.6500</td>\n",
       "      <td>3.5</td>\n",
       "      <td>3.8</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>321930</th>\n",
       "      <td>5444</td>\n",
       "      <td>9394</td>\n",
       "      <td>0.875</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2006-06-21</td>\n",
       "      <td>No matter how much unique artistry they bring ...</td>\n",
       "      <td>4.4375</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.5</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>151451</th>\n",
       "      <td>2631</td>\n",
       "      <td>4680</td>\n",
       "      <td>0.400</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2006-06-21</td>\n",
       "      <td>All I wanted was a magical remote control of m...</td>\n",
       "      <td>2.3000</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.6</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>701870</th>\n",
       "      <td>4327</td>\n",
       "      <td>17557</td>\n",
       "      <td>0.500</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>2006-06-21</td>\n",
       "      <td>When Jean Grey starts gasping, 'Kill me!' it's...</td>\n",
       "      <td>2.7500</td>\n",
       "      <td>2.5</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>178320 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        user_id  movie_id  review_score  sentiment  emotion review_date  \\\n",
       "253595     9296      7594         0.600          4        2  1800-01-01   \n",
       "72106      7403      2494         1.000          3        4  1800-01-01   \n",
       "112724     9296      3644         0.800          1        2  1800-01-01   \n",
       "299580     7403      8800         0.700          4        2  1800-01-01   \n",
       "245700     7629      7374         0.625          3        4  1800-01-01   \n",
       "...         ...       ...           ...        ...      ...         ...   \n",
       "566157     8589     14185         0.700          1        0  2006-06-21   \n",
       "563361     4281     14097         0.700          2        2  2006-06-21   \n",
       "321930     5444      9394         0.875          1        2  2006-06-21   \n",
       "151451     2631      4680         0.400          0        2  2006-06-21   \n",
       "701870     4327     17557         0.500          3        2  2006-06-21   \n",
       "\n",
       "                                           review_content  origin_rating_0.5  \\\n",
       "253595  It's exciting to see a British horror film wit...             3.2000   \n",
       "72106   A grimly seductive end-of-the-world thriller, ...             5.0000   \n",
       "112724  There aren't many British horror films you cou...             4.1000   \n",
       "299580  Coming out from behind Spike Lee's camera, Ern...             3.6500   \n",
       "245700  A comedy western loaded with chuckles that onl...             3.3125   \n",
       "...                                                   ...                ...   \n",
       "566157  ... leaves the comic-book super-villain melodr...             3.6500   \n",
       "563361  Singer's work is generally strong, but Superma...             3.6500   \n",
       "321930  No matter how much unique artistry they bring ...             4.4375   \n",
       "151451  All I wanted was a magical remote control of m...             2.3000   \n",
       "701870  When Jean Grey starts gasping, 'Kill me!' it's...             2.7500   \n",
       "\n",
       "        rating_0.5  origin_rating_1  rating_1  \n",
       "253595         3.0              3.4         3  \n",
       "72106          5.0              5.0         5  \n",
       "112724         4.0              4.2         4  \n",
       "299580         3.5              3.8         4  \n",
       "245700         3.0              3.5         4  \n",
       "...            ...              ...       ...  \n",
       "566157         3.5              3.8         4  \n",
       "563361         3.5              3.8         4  \n",
       "321930         4.0              4.5         4  \n",
       "151451         2.0              2.6         3  \n",
       "701870         2.5              3.0         3  \n",
       "\n",
       "[178320 rows x 11 columns]"
      ]
     },
     "execution_count": 138,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainset"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "68708aee-bf05-49ac-908f-90759e9d2371",
   "metadata": {},
   "source": [
    "#### 3-3. trainset, testset 저장 (GCMC 데이터용)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "id": "f35eff6a-40a7-481d-aaec-4a6bda9b9997",
   "metadata": {},
   "outputs": [],
   "source": [
    "trainset.to_csv(path+'l_trainset.csv', index=False)\n",
    "filtered_testset.to_csv(path+'l_testset.csv', index=False)\n",
    "removed_testset.to_csv(path+'l_testset_removed.csv', index=False)\n",
    "# testset.to_csv(path+'testset.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1eed8810-ef59-48d1-86a6-f780175dfd36",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}