prod2vec.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Anaconda2\\lib\\site-packages\\gensim\\utils.py:860: UserWarning: detected Windows; aliasing chunkize to chunkize_serial\n",
      "  warnings.warn(\"detected Windows; aliasing chunkize to chunkize_serial\")\n"
     ]
    }
   ],
   "source": [
    "import gensim\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "with open(\"./data/ts.txt\", \"rb\") as f:\n",
    "    product_ts = f.readlines()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "196 14084 12427 26088 26405 <EOO> 196 10258 12427 13176 26088 13032 <EOO> 196 12427 10258 25133 30450 <EOO> 196 12427 10258 25133 26405 <EOO> 196 12427 10258 25133 10326 17122 41787 13176 <EOO> 196 12427 10258 25133 <EOO> 196 10258 12427 25133 13032 <EOO> 12427 196 10258 25133 46149 49235 <EOO> 49235 46149 25133 196 10258 12427 <EOO> 196 46149 39657 38928 25133 10258 35951 13032 12427 <EOO> 196 25133 38928 26405 39657 10258 13032 26088 27845 49235 46149\r\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(product_ts[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "split_product_ts = []\n",
    "for prd in product_ts:\n",
    "    split_product_ts.append(prd.split())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['25745', '39581', '34287', '22565', '36393', '39832', '17743', '43409', '45596', '41473', '24838', '28199', '14778', '4137', '33894', '29432', '9508', '<EOO>', '9508', '45596', '14778', '33894', '3696', '33000', '39832', '31663', '10673', '25344', '4792', '36978', '41650', '43954', '13370', '47307', '<EOO>', '9508', '14778', '33894', '28199', '31663', '4792', '24799', '36978', '41650', '34287', '32650', '20561', '17982', '34023', '49279', '37158', '1609', '<EOO>', '33894', '14778', '28199', '36978', '4792', '17982', '24799', '45596', '39832', '37158', '17743', '20955', '30696', '16786', '35390', '28647', '<EOO>', '9508', '28199', '33894', '7606', '20955', '17339', '30597', '7168', '25266', '31964', '32813', '42976', '17982', '34862']\n"
     ]
    }
   ],
   "source": [
    "print(split_product_ts[1000])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# create word2vec model\n",
    "model = gensim.models.Word2Vec(split_product_ts, size=32, window=3, min_count=1, workers=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# save in binary format\n",
    "#gensim.models.Word2Vec.save_word2vec_format(model, './data/prod2vec_32.bin', binary=True)\n",
    "model.wv.save_word2vec_format('./data/prod2vec_32.vec')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# import products\n",
    "products = pd.read_csv(\"./data/products.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>product_id</th>\n",
       "      <th>product_name</th>\n",
       "      <th>aisle_id</th>\n",
       "      <th>department_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Chocolate Sandwich Cookies</td>\n",
       "      <td>61</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>All-Seasons Salt</td>\n",
       "      <td>104</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>Robust Golden Unsweetened Oolong Tea</td>\n",
       "      <td>94</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>Smart Ones Classic Favorites Mini Rigatoni Wit...</td>\n",
       "      <td>38</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>Green Chile Anytime Sauce</td>\n",
       "      <td>5</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   product_id                                       product_name  aisle_id  \\\n",
       "0           1                         Chocolate Sandwich Cookies        61   \n",
       "1           2                                   All-Seasons Salt       104   \n",
       "2           3               Robust Golden Unsweetened Oolong Tea        94   \n",
       "3           4  Smart Ones Classic Favorites Mini Rigatoni Wit...        38   \n",
       "4           5                          Green Chile Anytime Sauce         5   \n",
       "\n",
       "   department_id  \n",
       "0             19  \n",
       "1             13  \n",
       "2              7  \n",
       "3              1  \n",
       "4             13  "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "products.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('13176', 0.7737448215484619),\n",
       " ('49235', 0.7622644305229187),\n",
       " ('24852', 0.7421951293945312),\n",
       " ('25146', 0.7301254868507385),\n",
       " ('27086', 0.6835342645645142)]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# explore prod2vec model\n",
    "model.most_similar(positive=['196'], topn=5)\n",
    "## they are all sodas GREAT!!!\n",
    "model.most_similar(positive=['27845'], topn=5)\n",
    "## milk and yoghurt\n",
    "model.most_similar(positive=['13379'], topn=5)\n",
    "## noodles and sauce\n",
    "model.most_similar(positive=['<EOO>'], topn=5)\n",
    "## EOO: Banas are very similar -> is the most commen product (?)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['33894', 25976, 18721, 46448, 47946, 18792]\n"
     ]
    }
   ],
   "source": [
    "prd = '33894'\n",
    "topn = model.most_similar(positive=[prd], topn=5)\n",
    "topn_id = [int(p[0]) for p in topn]\n",
    "topn_id.insert(0, prd)\n",
    "print(topn_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "       product_id                                       product_name  \\\n",
      "18720       18721      Baked Rice and Corn Puffs, Aged White Cheddar   \n",
      "18791       18792              Goldfish Pretzel Baked Snack Crackers   \n",
      "25975       25976  Pepperidge Farm Colors Cheddar Baked Snack Cra...   \n",
      "33893       33894  Goldfish Cheddar Baked Snack Crackers Multi Packs   \n",
      "46447       46448  Pepperidge Farm Goldfish Neon Colors Baked Sna...   \n",
      "47945       47946                                      Nutella & Go!   \n",
      "\n",
      "       aisle_id  department_id  \n",
      "18720       107             19  \n",
      "18791        78             19  \n",
      "25975        78             19  \n",
      "33893        78             19  \n",
      "46447        78             19  \n",
      "47945        88             13  \n"
     ]
    }
   ],
   "source": [
    "prods_nbrs = products.loc[products['product_id'].isin(topn_id)]\n",
    "print(prods_nbrs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>product_id</th>\n",
       "      <th>product_name</th>\n",
       "      <th>aisle_id</th>\n",
       "      <th>department_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1167</th>\n",
       "      <td>1168</td>\n",
       "      <td>White Wine Winemaker's Blend California, 2010</td>\n",
       "      <td>62</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1941</th>\n",
       "      <td>1942</td>\n",
       "      <td>Aged White Wine Vinegar</td>\n",
       "      <td>19</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3875</th>\n",
       "      <td>3876</td>\n",
       "      <td>Sofia Blanc De Blancs Sparkling White Wine</td>\n",
       "      <td>134</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5297</th>\n",
       "      <td>5298</td>\n",
       "      <td>30 Minute Marinade Herb &amp; White Wine With Lemo...</td>\n",
       "      <td>5</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8094</th>\n",
       "      <td>8095</td>\n",
       "      <td>White Wine Salami, Pinot Grigio</td>\n",
       "      <td>96</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14286</th>\n",
       "      <td>14287</td>\n",
       "      <td>White Wine Vinegar</td>\n",
       "      <td>19</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18113</th>\n",
       "      <td>18114</td>\n",
       "      <td>White Wine &amp; Garlic Sauces Skillet Shrimp Scampi</td>\n",
       "      <td>5</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31051</th>\n",
       "      <td>31052</td>\n",
       "      <td>White Wine Chicken &amp; Couscous</td>\n",
       "      <td>100</td>\n",
       "      <td>21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31661</th>\n",
       "      <td>31662</td>\n",
       "      <td>Organic White Wine Vinegar</td>\n",
       "      <td>19</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33000</th>\n",
       "      <td>33001</td>\n",
       "      <td>White Wine</td>\n",
       "      <td>62</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37550</th>\n",
       "      <td>37551</td>\n",
       "      <td>Pinot Grigio White Wine Vinegar</td>\n",
       "      <td>19</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42627</th>\n",
       "      <td>42628</td>\n",
       "      <td>Garlic White Wine Shrimp Saute</td>\n",
       "      <td>34</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47573</th>\n",
       "      <td>47574</td>\n",
       "      <td>Most Wanted California Pinot Grigio White Wine</td>\n",
       "      <td>62</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47936</th>\n",
       "      <td>47937</td>\n",
       "      <td>White Wine and Balsamic Vinegar Flavored Beets</td>\n",
       "      <td>123</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       product_id                                       product_name  \\\n",
       "1167         1168      White Wine Winemaker's Blend California, 2010   \n",
       "1941         1942                            Aged White Wine Vinegar   \n",
       "3875         3876         Sofia Blanc De Blancs Sparkling White Wine   \n",
       "5297         5298  30 Minute Marinade Herb & White Wine With Lemo...   \n",
       "8094         8095                    White Wine Salami, Pinot Grigio   \n",
       "14286       14287                                 White Wine Vinegar   \n",
       "18113       18114   White Wine & Garlic Sauces Skillet Shrimp Scampi   \n",
       "31051       31052                      White Wine Chicken & Couscous   \n",
       "31661       31662                         Organic White Wine Vinegar   \n",
       "33000       33001                                         White Wine   \n",
       "37550       37551                    Pinot Grigio White Wine Vinegar   \n",
       "42627       42628                     Garlic White Wine Shrimp Saute   \n",
       "47573       47574     Most Wanted California Pinot Grigio White Wine   \n",
       "47936       47937     White Wine and Balsamic Vinegar Flavored Beets   \n",
       "\n",
       "       aisle_id  department_id  \n",
       "1167         62              5  \n",
       "1941         19             13  \n",
       "3875        134              5  \n",
       "5297          5             13  \n",
       "8094         96             20  \n",
       "14286        19             13  \n",
       "18113         5             13  \n",
       "31051       100             21  \n",
       "31661        19             13  \n",
       "33000        62              5  \n",
       "37550        19             13  \n",
       "42627        34              1  \n",
       "47573        62              5  \n",
       "47936       123              4  "
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## find product IDs for specific words\n",
    "products.loc[products['product_name'].str.contains(\"white wine\", case = False)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.93431654434436595"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Woman is to man as king is to ...\n",
    "#model.most_similar(positive=['woman', 'king'], negative=['man'])\n",
    "\n",
    "# Kellog's cereal (6213) is to milk (27845) as pasta (33) is to ...\n",
    "model.most_similar(positive=['6213', '33'], negative=['27845'])\n",
    "\n",
    "# Bud Light (41816) is to Doritos (2144) as white wine (33001) is to ...\n",
    "model.most_similar(positive=['33001', '41816'], negative=['2144'])\n",
    "\n",
    "## doesnt work too well yet...\n",
    "\n",
    "# How similar are two particular products according to the model?\n",
    "model.similarity(\"598\", \"33001\")\n",
    "## Cabernet Sauvignon and white wine\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [Root]",
   "language": "python",
   "name": "Python [Root]"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}