recomend_movie.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "recomend_movie.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Yongtae723/movie_recomendation_1/blob/master/recomend_movie.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TkVBtHzUcP3e",
        "colab_type": "text"
      },
      "source": [
        "##映画のおすすめを探すプログラム\n",
        "この[ブログ](http://www.analyze-world.com/entry/2017/11/21/084654)のほぼ写経\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EAR5KO4QdOwk",
        "colab_type": "text"
      },
      "source": [
        "#スクレイピング\n",
        "流れ的にはビュアーの情報をinput\n",
        "その後映画の評価のデータをinput\n",
        "\n",
        "使うサイトは[みんなのシネマレビュー](https://www.jtnews.jp/index_o.html)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "95MsLs1OdmNt",
        "colab_type": "text"
      },
      "source": [
        "## まずはレビュアーの情報取得"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IR_znCpHcPXI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#必要なライブラリをインポート\n",
        "from bs4 import BeautifulSoup #requestsで取得したデータをうまく読み取るのに必要なライブラリ\n",
        "import requests\n",
        "import pandas as pd\n",
        "from pandas import Series, DataFrame\n",
        "import time\n",
        "from tqdm import tqdm "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xe5sRgbAcjly",
        "colab_type": "code",
        "outputId": "3a972ea9-4ed3-4264-93e3-0edbbaa5c49b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 109
        }
      },
      "source": [
        "#レビュアーのID、名前、レビュー数、最終投稿日を取得\n",
        "\n",
        "\n",
        "\n",
        "#URL（レビュアーリストの1ページ目）\n",
        "reviewer_url = 'https://www.jtnews.jp/cgi-bin_o/revlist.cgi?PAGE_NO='\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['20001', '20031', '20033', '20035', '20036', '20037', '20038', '20039', '20040', '20041', '20042', '20043', '20044', '20045', '20047', '20048', '20049', '20050', '20051', '20053', '20054', '20055', '20056', '20057', '20060', '20061', '20063', '20064', '20065', '20066', '20067', '20068', '20070', '20071', '20073', '20074', '20075', '20077', '20078', '20080']\n",
            "['シネマレビュー管理人', 'モートルの玉', 'スマイル・ペコ', 'azuncha03', 'ロバちゃん', 'GreenTV', 'InVincible', 'もみじプリン', 'へっぽこ', 'イマジン', 'やましんの巻', 'wood', '神父', '--------', '井尻又兵衛', 'なるせたろう', 'じゃん＋＋', '由布', 'カズゥー柔術', 'すぎさ', 'チョコレクター', '法子', 'ヤスピ', 'tantan', 'Ｍ・Ｒ・サイケデリコン', 'ハイラムK', 'ss', '鉄男', '山岳蘭人', 'ぐるぐる', 'あろえりーな', '秀吉', '_', 'たけおん', 'まるけん', 'ドラえもん', 'kazu_jon', 'kazunabe', 'rain', 'オオカミ']\n",
            "['168', '139', '813', '59', '81', '3', '0', '320', '125', '1510', '729', '680', '98', '0', '3', '162', '250', '163', '621', '122', '395', '151', '73', '983', '3578', '0', '4', '165', '254', '1305', '4355', '49', '0', '9', '18', '903', '79', '29', '25', '1070']\n",
            "['2019-05-05', '2003-12-06', '2003-10-16', '2009-12-10', '2003-10-14', '2003-07-21', '2004-06-25', '2004-03-12', '2007-05-07', '2003-10-07', '2016-03-29', '2017-05-21', '2004-08-22', '2006-03-17', '2003-08-03', '2007-06-15', '2013-01-17', '2007-12-20', '2010-07-16', '2007-01-28', '2019-04-12', '2008-07-28', '2004-05-25', '2008-11-12', '2019-05-08', '2006-07-20', '2003-09-04', '2012-09-24', '2004-09-09', '2011-08-28', '2019-05-11', '2013-08-17', '2005-09-28', '2003-07-07', '2004-05-15', '2013-04-06', '2009-06-24', '2005-08-23', '2004-01-17', '2016-03-03']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Egts2fg7qWHq",
        "colab_type": "code",
        "outputId": "02a58177-8bf5-477c-b606-427802a5113e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 127
        }
      },
      "source": [
        "#まずは1ページ目を見て感触をつかむ\n",
        "\n",
        "#リストを用意\n",
        "ID1_list = []\n",
        "Name_list = []\n",
        "nReview_list = []\n",
        "lastReview_list = []\n",
        "\n",
        "reviewer_url_no = reviewer_url + str(1)\n",
        "\n",
        "#データ取得\n",
        "result = requests.get(reviewer_url_no)\n",
        "c = result.content\n",
        "\n",
        "\n",
        "#HTMLを元に、オブジェクトを作る\n",
        "soup = BeautifulSoup(c, \"lxml\")\n",
        "\n",
        "#リストの部分を切り出し\n",
        "summary = soup.find_all(\"td\",{'valign':'TOP'})\n",
        "\n",
        "#2つのテーブルを抜き出し\n",
        "tables = summary[2].find_all('table',{'bgcolor':'#4499FF'})\n",
        "\n",
        "#レビュアーリストの方からtrを抜き出し\n",
        "trs = tables[1].find_all('tr')\n",
        "\n",
        "#ページ内のレビュアー情報をループして取得\n",
        "for i in tqdm(range(2,len(trs))):\n",
        "  ths = trs[i].find('th') #No\n",
        "  tds = trs[i].find_all('td') #名前、レビュー数、最終レビュー日\n",
        "\n",
        "  # #Noを取得\n",
        "  No = str(ths)\n",
        "  No = No.replace('<th><font color=\"GREEN\">','')\n",
        "  No = No.replace('</font></th>','')\n",
        "  ID1_list.append(No)\n",
        "  Replace_str = '<td><a href=\"revper.cgi?&amp;REVPER_NO=' + No + '\">' #名前部分から文字列を削除するために用意\n",
        "\n",
        "\n",
        "  # #名前を取得\n",
        "  Name = str(tds[0])\n",
        "  Name = Name.replace('</a>さん</td>','')\n",
        "  Name = Name.replace(Replace_str, '')\n",
        "  Name_list.append(Name)\n",
        "\n",
        "  # #レビュー数を取得\n",
        "  nReview = str(tds[1])\n",
        "  nReview = nReview.replace('<td>','')\n",
        "  nReview = nReview.replace('</td>','')\n",
        "  nReview_list.append(nReview)\n",
        "\n",
        "  # #最終レビュー日を取得\n",
        "  lastReview = str(tds[2])\n",
        "  lastReview = lastReview.replace('<td>','')\n",
        "  lastReview = lastReview.replace('</td>','')\n",
        "  lastReview_list.append(lastReview)\n",
        "\n",
        "  \n",
        "print(ID1_list)\n",
        "print(Name_list)\n",
        "print(nReview_list)\n",
        "print(lastReview_list)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 40/40 [00:00<00:00, 4741.87it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "['20001', '20031', '20033', '20035', '20036', '20037', '20038', '20039', '20040', '20041', '20042', '20043', '20044', '20045', '20047', '20048', '20049', '20050', '20051', '20053', '20054', '20055', '20056', '20057', '20060', '20061', '20063', '20064', '20065', '20066', '20067', '20068', '20070', '20071', '20073', '20074', '20075', '20077', '20078', '20080']\n",
            "['シネマレビュー管理人', 'モートルの玉', 'スマイル・ペコ', 'azuncha03', 'ロバちゃん', 'GreenTV', 'InVincible', 'もみじプリン', 'へっぽこ', 'イマジン', 'やましんの巻', 'wood', '神父', '--------', '井尻又兵衛', 'なるせたろう', 'じゃん＋＋', '由布', 'カズゥー柔術', 'すぎさ', 'チョコレクター', '法子', 'ヤスピ', 'tantan', 'Ｍ・Ｒ・サイケデリコン', 'ハイラムK', 'ss', '鉄男', '山岳蘭人', 'ぐるぐる', 'あろえりーな', '秀吉', '_', 'たけおん', 'まるけん', 'ドラえもん', 'kazu_jon', 'kazunabe', 'rain', 'オオカミ']\n",
            "['168', '139', '813', '59', '81', '3', '0', '320', '125', '1510', '729', '680', '98', '0', '3', '162', '250', '163', '621', '122', '395', '151', '73', '983', '3578', '0', '4', '165', '254', '1305', '4355', '49', '0', '9', '18', '903', '79', '29', '25', '1070']\n",
            "['2019-05-05', '2003-12-06', '2003-10-16', '2009-12-10', '2003-10-14', '2003-07-21', '2004-06-25', '2004-03-12', '2007-05-07', '2003-10-07', '2016-03-29', '2017-05-21', '2004-08-22', '2006-03-17', '2003-08-03', '2007-06-15', '2013-01-17', '2007-12-20', '2010-07-16', '2007-01-28', '2019-04-12', '2008-07-28', '2004-05-25', '2008-11-12', '2019-05-08', '2006-07-20', '2003-09-04', '2012-09-24', '2004-09-09', '2011-08-28', '2019-05-11', '2013-08-17', '2005-09-28', '2003-07-07', '2004-05-15', '2013-04-06', '2009-06-24', '2005-08-23', '2004-01-17', '2016-03-03']\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "s8Zzh7UOlnhA",
        "colab_type": "code",
        "outputId": "5bd52ee1-0119-4a6e-fdf4-478689867fe7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "#悪くなかったので、すべてのレビューアーの情報を取得\n",
        "\n",
        "\n",
        "#リストを用意\n",
        "ID1_list = []\n",
        "Name_list = []\n",
        "nReview_list = []\n",
        "lastReview_list = []\n",
        "\n",
        "\n",
        "for page_num in tqdm(range(1,135)):\n",
        "  reviewer_url_no = reviewer_url + str(1)\n",
        "\n",
        "  #データ取得\n",
        "  result = requests.get(reviewer_url_no)\n",
        "  c = result.content\n",
        "\n",
        "\n",
        "  #HTMLを元に、オブジェクトを作る\n",
        "  soup = BeautifulSoup(c, \"lxml\")\n",
        "\n",
        "  #リストの部分を切り出し\n",
        "  summary = soup.find_all(\"td\",{'valign':'TOP'})\n",
        "\n",
        "  #2つのテーブルを抜き出し\n",
        "  tables = summary[2].find_all('table',{'bgcolor':'#4499FF'})\n",
        "\n",
        "  #レビュアーリストの方からtrを抜き出し\n",
        "  trs = tables[1].find_all('tr')\n",
        "\n",
        "  #ページ内のレビュアー情報をループして取得\n",
        "  for i in range(2,len(trs)):\n",
        "    ths = trs[i].find('th') #No\n",
        "    tds = trs[i].find_all('td') #名前、レビュー数、最終レビュー日\n",
        "\n",
        "    # #Noを取得\n",
        "    No = str(ths)\n",
        "    No = No.replace('<th><font color=\"GREEN\">','')\n",
        "    No = No.replace('</font></th>','')\n",
        "    ID1_list.append(No)\n",
        "    Replace_str = '<td><a href=\"revper.cgi?&amp;REVPER_NO=' + No + '\">' #名前部分から文字列を削除するために用意\n",
        "\n",
        "\n",
        "    # #名前を取得\n",
        "    Name = str(tds[0])\n",
        "    Name = Name.replace('</a>さん</td>','')\n",
        "    Name = Name.replace(Replace_str, '')\n",
        "    Name_list.append(Name)\n",
        "\n",
        "    # #レビュー数を取得\n",
        "    nReview = str(tds[1])\n",
        "    nReview = nReview.replace('<td>','')\n",
        "    nReview = nReview.replace('</td>','')\n",
        "    nReview_list.append(nReview)\n",
        "\n",
        "    # #最終レビュー日を取得\n",
        "    lastReview = str(tds[2])\n",
        "    lastReview = lastReview.replace('<td>','')\n",
        "    lastReview = lastReview.replace('</td>','')\n",
        "    lastReview_list.append(lastReview)\n",
        "    \n",
        "  time.sleep(3) #待機"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 134/134 [08:42<00:00,  3.87s/it]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CHqqOC0VtWh4",
        "colab_type": "code",
        "outputId": "4d58ae40-2cbd-4a90-dda5-4101ad6ff23c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 89
        }
      },
      "source": [
        "print(len(ID1_list))\n",
        "print(len(Name_list))\n",
        "print(len(nReview_list))\n",
        "print(len(lastReview_list))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "5360\n",
            "5360\n",
            "5360\n",
            "5360\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tZLfaWFk0tN7",
        "colab_type": "text"
      },
      "source": [
        "###上のデータの読み込みが時間かかりまくったので、Pickle形式で保存し次からこのデータを直で読み込むようにする。\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rerRNY7Nvi47",
        "colab_type": "code",
        "outputId": "51bc2b9a-10b0-417f-92df-8db86bf5f332",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 127
        }
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3Aietf%3Awg%3Aoauth%3A2.0%3Aoob&scope=email%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdocs.test%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive.photos.readonly%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fpeopleapi.readonly&response_type=code\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "n91aOZhlv5zp",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import os\n",
        "# os.chdir('./drive/My Drive')\n",
        "os.mkdir('movie_review_Data_set')\n",
        "os.chdir('movie_review_Data_set')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dhBCFkHsvgSg",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import pickle\n",
        "\n",
        "pickle_out = open('ID1_list_2019_5_12.pickle', 'wb')\n",
        "pickle.dump(ID1_list, pickle_out)\n",
        "pickle_out.close()\n",
        "\n",
        "pickle_out = open('Name_list_2019_5_12.pickle', 'wb')\n",
        "pickle.dump(Name_list, pickle_out)\n",
        "pickle_out.close()\n",
        "\n",
        "pickle_out = open('nReview_list_2019_5_12.pickle', 'wb')\n",
        "pickle.dump(nReview_list, pickle_out)\n",
        "pickle_out.close()\n",
        "\n",
        "pickle_out = open('lastReview_list_2019_5_12.pickle', 'wb')\n",
        "pickle.dump(lastReview_list, pickle_out)\n",
        "pickle_out.close()\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8WOqCdho0xTI",
        "colab_type": "text"
      },
      "source": [
        "###  このセルは12時間以上かかるため、lacal パソコンのjupyter notebookで解析する"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "R6T684roq20I",
        "colab_type": "code",
        "outputId": "93dc905d-c3b7-47de-8744-0b7d43816891",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 243
        }
      },
      "source": [
        "# #レビュアーの性別と年齢を取得\n",
        "\n",
        "\n",
        "\n",
        "# #リストを用意\n",
        "# gender_list = []\n",
        "# age_list = []\n",
        "\n",
        "# url = 'https://www.jtnews.jp/cgi-bin_o/revper.cgi?&REVPER_NO='#それぞれのレビュワーのweb pageはidで管理されている\n",
        "\n",
        "\n",
        "# #まずは1ページでやってみる\n",
        "# for ID in tqdm(ID1_list):\n",
        "#   individual_first_url = url + str(ID)\n",
        "\n",
        "#   #データ取得\n",
        "#   result = requests.get(individual_first_url)\n",
        "#   c = result.content\n",
        "\n",
        "#   #HTMLを元に、オブジェクトを作る\n",
        "#   soup = BeautifulSoup(c, \"lxml\")\n",
        "\n",
        "#   #リストの部分を切り出し\n",
        "#   summary = soup.find_all(\"td\",{'valign':'TOP'})\n",
        "\n",
        "#   #2つのテーブルを抜き出し\n",
        "#   tables = summary[2].find_all('table',{'bgcolor':'#4499FF'})\n",
        "\n",
        "#   #プロフィールのテーブル(tables[0])からtrを抜き出し\n",
        "#   fonts = tables[0].find_all('font', {'color':'GREEN'})\n",
        "\n",
        "  \n",
        "  \n",
        "#   #年齢と性別は記入してない人がいるので記入ありと無しで区別する\n",
        "#   gender = False\n",
        "#   age = False\n",
        "\n",
        "#   for font in fonts:\n",
        "#       if '性別' in font:\n",
        "#           gender = True\n",
        "#       if '年齢' in font:\n",
        "#           age = True\n",
        "\n",
        "#   #プロフィールの方からtrを抜き出し\n",
        "#   trs = tables[0].find_all('tr', {'bgcolor':'#FFFFFF'})\n",
        "\n",
        "#   for tr in trs:\n",
        "#       tr = str(tr)\n",
        "#       if '性別' in tr:\n",
        "#           gender = tr.replace(\n",
        "#               '<tr bgcolor=\"#FFFFFF\"><th align=\"LEFT\"><font color=\"GREEN\">性別</font></th><td>\\r\\n','')\n",
        "#           gender = gender.replace(\n",
        "#               '<tr bgcolor=\"#FFFFFF\"><th align=\"LEFT\"><font color=\"GREEN\">性別</font></th><td>\\n','')\n",
        "#           gender = gender.replace('</td></tr>','')\n",
        "#           gender_list.append(gender)\n",
        "#       if '年齢' in tr:\n",
        "#           age = tr.replace(\n",
        "#               '<tr bgcolor=\"#FFFFFF\"><th align=\"LEFT\"><font color=\"GREEN\">年齢</font></th><td>\\r\\n','')\n",
        "#           age = age.replace('</td></tr>','')\n",
        "#           age_list.append(age)\n",
        "#       time.sleep(1) #待機\n",
        "\n",
        "#   if gender == False:\n",
        "#       gender_list.append('')\n",
        "#   if age == False:\n",
        "#       age_list.append('')\n",
        "\n",
        "#   time.sleep(5) #待機"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "  0%|          | 0/5360 [00:00<?, ?it/s]\u001b[A"
          ],
          "name": "stderr"
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-70-b8f98a8a3b87>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     61\u001b[0m       \u001b[0mage_list\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m''\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     62\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 63\u001b[0;31m   \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msleep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m#待機\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "p_YzaB3WxX0H",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}