hyper-mega-ultra-wtf-graph-tool.ipynb

{"cells":[{"cell_type":"code","execution_count":1,"metadata":{"id":"O5fNbhRPoCt1"},"outputs":[],"source":["%load_ext autoreload \n","%autoreload 2\n"]},{"cell_type":"code","execution_count":2,"metadata":{"id":"vGdAcZvZGdjA"},"outputs":[],"source":["\n","#lb\n","import pandas as pd\n","edges = pd.concat([\n","    pd.read_parquet(f'/home/anhphantq/unidata_sub/edges_sub_frag_{i}_v1.parquet') for i in range (64) \n","    ])\n","edges_test = pd.read_parquet('/home/anhphantq/unidata_sub/edges_test_sub_v1.parquet')\n","\n","vertices = pd.read_parquet('/home/anhphantq/unidata_sub/train_vprops_v1.parquet')\n","vertices_test = pd.read_parquet('/home/anhphantq/unidata_sub/test_vprops_v1.parquet')\n","\n","item_agg = pd.read_parquet('/home/anhphantq/unidata_sub/item_props_v1.parquet')\n","item_glob = pd.read_parquet('/home/anhphantq/unidata_sub/item_glob_v1.parquet')\n","\n","user_features_df = pd.read_parquet('/home/anhphantq/unidata_sub/user_features.parquet')\n","item_features_df = pd.read_parquet('/home/anhphantq/unidata_sub/item_features.parquet')"]},{"cell_type":"code","execution_count":2,"metadata":{"id":"mj3SQPqbi8TT"},"outputs":[],"source":["\n","#valid\n","import pandas as pd\n","edges = pd.concat([\n","    pd.read_parquet(f'/home/anhphantq/splitted_unidata_sub/edges_sub_frag_{i}_v1.parquet') for i in range (64) \n","    ])\n","edges_test = pd.read_parquet('/home/anhphantq/splitted_unidata_sub/edges_test_sub_v1.parquet')\n","\n","vertices = pd.read_parquet('/home/anhphantq/splitted_unidata_sub/train_vprops_v1.parquet')\n","vertices_test = pd.read_parquet('/home/anhphantq/splitted_unidata_sub/test_vprops_v1.parquet')\n","item_agg = pd.read_parquet('/home/anhphantq/splitted_unidata_sub/item_props_v1.parquet')\n","item_glob = pd.read_parquet('/home/anhphantq/splitted_unidata_sub/item_glob_v1.parquet')\n","\n","user_features_df = pd.read_parquet('/home/anhphantq/splitted_unidata_sub/user_features.parquet')\n","item_features_df = pd.read_parquet('/home/anhphantq/splitted_unidata_sub/item_features.parquet')"]},{"cell_type":"code","execution_count":3,"metadata":{},"outputs":[{"data":{"text/plain":["(0, 1814400, 92319733, 2419197)"]},"execution_count":3,"metadata":{},"output_type":"execute_result"}],"source":["MIN_TS = edges.iloc[:, 2].min()\n","MAX_TS = edges.iloc[:, 2].max() + 1\n","MAX_TEST_TS = edges_test.iloc[:, 2].max() + 1\n","NUM_AIDS = edges.iloc[:, 1].max() + 1\n","assert edges.iloc[:, 1].min() == 0\n","NUM_TRAIN_EDGES = edges.shape[0]\n","MIN_TS, MAX_TS, NUM_TRAIN_EDGES, MAX_TEST_TS"]},{"cell_type":"code","execution_count":4,"metadata":{},"outputs":[],"source":["from meow_utils import feature_id_map\n","import numpy as np \n","disk_vp = np.fromfile('splitted_vertice_properties.array').reshape(len(feature_id_map), -1)\n","disk_vp.setflags(write = False)"]},{"cell_type":"code","execution_count":4,"metadata":{},"outputs":[],"source":["from meow_utils import feature_id_map\n","import numpy as np \n","disk_vp = np.fromfile('vertice_properties.array').reshape(len(feature_id_map), -1)\n","disk_vp.setflags(write = False)"]},{"cell_type":"code","execution_count":5,"metadata":{"id":"9oJ-8glpRNTC"},"outputs":[],"source":["JOIN = True"]},{"cell_type":"code","execution_count":5,"metadata":{},"outputs":[],"source":["JOIN = False"]},{"cell_type":"code","execution_count":6,"metadata":{"executionInfo":{"elapsed":279,"status":"ok","timestamp":1672119444295,"user":{"displayName":"Hải Minh Nguyễn","userId":"16371132940967469863"},"user_tz":-420},"id":"ZZMsFTaRbDfA"},"outputs":[],"source":["#Params\n","import numpy as np\n","TEST_CLICK_WEIGHT = 3\n","TEST_CART_WEIGHT = 6\n","TEST_ORDER_WEIGHT_L= 6\n","TEST_ORDER_WEIGHT_S = 2\n","\n","MAX_NUM_CANDS = 50\n","\n","MIN_TOPK_COUSER = 1000\n","COFITNESS_THRES = 0\n","\n","\n","ALPHA = 0.5\n","BETA = 0.5\n","FILTER = False\n","\n","#PARAMS\n","\n","get_test_items_params = {\n","      'test_click_weight': TEST_CLICK_WEIGHT,\n","      'test_cart_weight': TEST_CART_WEIGHT,\n","      'test_order_weight_l': TEST_ORDER_WEIGHT_L,\n","      'test_order_weight_s': TEST_ORDER_WEIGHT_S,\n","      'day' : 3600 * 8,  \n","  }\n","\n","get_items_params = {\n","      'click_weight': 3,\n","      'cart_weight' : 10,\n","      'order_weight': 1,\n","      'day' : 3600 * 8,  \n","    }\n","\n"]},{"cell_type":"markdown","metadata":{"id":"qcYhDDnhj_AJ","notebookRunGroups":{"groupValue":"1"}},"source":["# Create edges and vertices"]},{"cell_type":"code","execution_count":7,"metadata":{"id":"b-UH6WbTORwb","notebookRunGroups":{"groupValue":""}},"outputs":[],"source":["first_test_session = edges_test.iloc[0,0]"]},{"cell_type":"code","execution_count":8,"metadata":{"id":"g3slAAcCHH0Z"},"outputs":[],"source":["all_edges = pd.concat((edges, edges_test))"]},{"cell_type":"code","execution_count":9,"metadata":{"id":"Ps4A9PrwIqZT"},"outputs":[],"source":["del edges, edges_test"]},{"cell_type":"code","execution_count":10,"metadata":{"id":"pCx10jpk0rML"},"outputs":[],"source":["all_vertices = pd.concat((vertices, vertices_test))"]},{"cell_type":"code","execution_count":11,"metadata":{"id":"LlxVTJqp0vN5"},"outputs":[],"source":["del vertices, vertices_test"]},{"cell_type":"code","execution_count":12,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":30,"status":"ok","timestamp":1672106986686,"user":{"displayName":"Hải Minh Nguyễn","userId":"16371132940967469863"},"user_tz":-420},"id":"Wf_37nkLHUPR","outputId":"96fb12d0-5af4-4397-83a7-0e6d9ac661a7"},"outputs":[{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>('session', '')</th>\n","      <th>('aid', '')</th>\n","      <th>('ts', 'last')</th>\n","      <th>('ts', 'first')</th>\n","      <th>('clicks', 'sum')</th>\n","      <th>('carts', 'sum')</th>\n","      <th>('orders', 'sum')</th>\n","      <th>('subsession', 'encode')</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>0</td>\n","      <td>16246</td>\n","      <td>62919</td>\n","      <td>62919</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>2</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>0</td>\n","      <td>30373</td>\n","      <td>1798887</td>\n","      <td>1798887</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>2147483648</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>0</td>\n","      <td>97836</td>\n","      <td>1590509</td>\n","      <td>1590509</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>268435456</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>0</td>\n","      <td>102416</td>\n","      <td>1714839</td>\n","      <td>1714839</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>1073741824</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>0</td>\n","      <td>154930</td>\n","      <td>1241409</td>\n","      <td>1241409</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>524288</td>\n","    </tr>\n","    <tr>\n","      <th>...</th>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","    </tr>\n","    <tr>\n","      <th>5535985</th>\n","      <td>12899774</td>\n","      <td>33035</td>\n","      <td>2419168</td>\n","      <td>2419168</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>5535986</th>\n","      <td>12899775</td>\n","      <td>1743151</td>\n","      <td>2419170</td>\n","      <td>2419170</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>5535987</th>\n","      <td>12899776</td>\n","      <td>548599</td>\n","      <td>2419172</td>\n","      <td>2419172</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>5535988</th>\n","      <td>12899777</td>\n","      <td>384045</td>\n","      <td>2419176</td>\n","      <td>2419176</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>5535989</th>\n","      <td>12899778</td>\n","      <td>561560</td>\n","      <td>2419183</td>\n","      <td>2419183</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>1</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>97855723 rows × 8 columns</p>\n","</div>"],"text/plain":["         ('session', '')  ('aid', '')  ('ts', 'last')  ('ts', 'first')  \\\n","0                      0        16246           62919            62919   \n","1                      0        30373         1798887          1798887   \n","2                      0        97836         1590509          1590509   \n","3                      0       102416         1714839          1714839   \n","4                      0       154930         1241409          1241409   \n","...                  ...          ...             ...              ...   \n","5535985         12899774        33035         2419168          2419168   \n","5535986         12899775      1743151         2419170          2419170   \n","5535987         12899776       548599         2419172          2419172   \n","5535988         12899777       384045         2419176          2419176   \n","5535989         12899778       561560         2419183          2419183   \n","\n","         ('clicks', 'sum')  ('carts', 'sum')  ('orders', 'sum')  \\\n","0                        1                 0                  0   \n","1                        1                 0                  0   \n","2                        1                 0                  0   \n","3                        1                 0                  0   \n","4                        1                 0                  0   \n","...                    ...               ...                ...   \n","5535985                  1                 0                  0   \n","5535986                  1                 0                  0   \n","5535987                  1                 0                  0   \n","5535988                  1                 0                  0   \n","5535989                  1                 0                  0   \n","\n","         ('subsession', 'encode')  \n","0                               2  \n","1                      2147483648  \n","2                       268435456  \n","3                      1073741824  \n","4                          524288  \n","...                           ...  \n","5535985                         1  \n","5535986                         1  \n","5535987                         1  \n","5535988                         1  \n","5535989                         1  \n","\n","[97855723 rows x 8 columns]"]},"execution_count":12,"metadata":{},"output_type":"execute_result"}],"source":["all_edges"]},{"cell_type":"code","execution_count":13,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":29,"status":"ok","timestamp":1672106986687,"user":{"displayName":"Hải Minh Nguyễn","userId":"16371132940967469863"},"user_tz":-420},"id":"5ZZ3Yp6AcW4E","outputId":"ac22e183-8c8c-420d-b4f1-2017e99cfd52"},"outputs":[{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>('session', '')</th>\n","      <th>('ts', 'min')</th>\n","      <th>('ts', 'max')</th>\n","      <th>('ts', 'count')</th>\n","      <th>('subsession', 'max')</th>\n","      <th>('consistency', '')</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>0</td>\n","      <td>1659304800</td>\n","      <td>1661103727</td>\n","      <td>147</td>\n","      <td>31</td>\n","      <td>1798927</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>1</td>\n","      <td>1659304800</td>\n","      <td>1660857067</td>\n","      <td>27</td>\n","      <td>6</td>\n","      <td>1552267</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>2</td>\n","      <td>1659304800</td>\n","      <td>1660577379</td>\n","      <td>13</td>\n","      <td>2</td>\n","      <td>1272579</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>3</td>\n","      <td>1659304800</td>\n","      <td>1661109666</td>\n","      <td>142</td>\n","      <td>22</td>\n","      <td>1804866</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>4</td>\n","      <td>1659304800</td>\n","      <td>1659304900</td>\n","      <td>3</td>\n","      <td>0</td>\n","      <td>100</td>\n","    </tr>\n","    <tr>\n","      <th>...</th>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","    </tr>\n","    <tr>\n","      <th>1801246</th>\n","      <td>12899774</td>\n","      <td>1661723968</td>\n","      <td>1661723968</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>1801247</th>\n","      <td>12899775</td>\n","      <td>1661723970</td>\n","      <td>1661723970</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>1801248</th>\n","      <td>12899776</td>\n","      <td>1661723972</td>\n","      <td>1661723972</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>1801249</th>\n","      <td>12899777</td>\n","      <td>1661723976</td>\n","      <td>1661723976</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>1801250</th>\n","      <td>12899778</td>\n","      <td>1661723983</td>\n","      <td>1661723983</td>\n","      <td>1</td>\n","      <td>0</td>\n","      <td>0</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>12899779 rows × 6 columns</p>\n","</div>"],"text/plain":["         ('session', '')  ('ts', 'min')  ('ts', 'max')  ('ts', 'count')  \\\n","0                      0     1659304800     1661103727              147   \n","1                      1     1659304800     1660857067               27   \n","2                      2     1659304800     1660577379               13   \n","3                      3     1659304800     1661109666              142   \n","4                      4     1659304800     1659304900                3   \n","...                  ...            ...            ...              ...   \n","1801246         12899774     1661723968     1661723968                1   \n","1801247         12899775     1661723970     1661723970                1   \n","1801248         12899776     1661723972     1661723972                1   \n","1801249         12899777     1661723976     1661723976                1   \n","1801250         12899778     1661723983     1661723983                1   \n","\n","         ('subsession', 'max')  ('consistency', '')  \n","0                           31              1798927  \n","1                            6              1552267  \n","2                            2              1272579  \n","3                           22              1804866  \n","4                            0                  100  \n","...                        ...                  ...  \n","1801246                      0                    0  \n","1801247                      0                    0  \n","1801248                      0                    0  \n","1801249                      0                    0  \n","1801250                      0                    0  \n","\n","[12899779 rows x 6 columns]"]},"execution_count":13,"metadata":{},"output_type":"execute_result"}],"source":["all_vertices"]},{"cell_type":"code","execution_count":14,"metadata":{"id":"UYSJagTi4JkR"},"outputs":[],"source":["MAX_SESSION = all_vertices.iloc[:,0].max()  + 1\n","all_edges.iloc[:,1] = all_edges.iloc[:,1] + MAX_SESSION\n"]},{"cell_type":"code","execution_count":15,"metadata":{"id":"zGyAc1YG9OdB"},"outputs":[],"source":["all_vertices = all_vertices.values"]},{"cell_type":"code","execution_count":16,"metadata":{"id":"Hh_j6ToH9ipb"},"outputs":[],"source":["import numpy as np\n","all_edges = all_edges.values"]},{"cell_type":"markdown","metadata":{"id":"U7rFWhPZLfk5"},"source":["# Load to graph"]},{"cell_type":"code","execution_count":17,"metadata":{"id":"5EaoxaS97fgU"},"outputs":[],"source":["import graph_tool"]},{"cell_type":"code","execution_count":18,"metadata":{"id":"Fde0RsKQToOD"},"outputs":[],"source":["from graph_tool import Graph\n","ug = Graph(directed=False)"]},{"cell_type":"code","execution_count":19,"metadata":{"id":"IJ47mX03UfyM"},"outputs":[],"source":["vmap = ug.add_edge_list(all_edges[:, : 2].astype(np.uint32))"]},{"cell_type":"code","execution_count":20,"metadata":{},"outputs":[{"data":{"text/plain":["(14755382, 97855723)"]},"execution_count":20,"metadata":{},"output_type":"execute_result"}],"source":["ug.num_vertices(), ug.num_edges()"]},{"cell_type":"code","execution_count":21,"metadata":{"id":"kn3Uld4oENef"},"outputs":[],"source":["#short type\n","all_edges[:, 4 : 7] = np.clip (all_edges[:, 4 : 7], 0,  32767)"]},{"cell_type":"code","execution_count":22,"metadata":{"id":"lw0XOjgdI8GF"},"outputs":[],"source":["data_type = {\n","    'elts': 'unsigned int',\n","    'efts': 'unsigned int',\n","    'eclick': 'unsigned int',\n","    'ecart': 'unsigned int',\n","    'eorder': 'unsigned int',\n","    'esub': 'unsigned long'\n","}"]},{"cell_type":"code","execution_count":23,"metadata":{"id":"_vsomv8Z0N-O"},"outputs":[],"source":["for i, prop in enumerate(['elts', 'efts', 'eclick', 'ecart', 'eorder', 'esub']):\n","  setattr(ug.ep, prop, ug.new_ep(data_type[prop], vals = all_edges[:, i + 2]))  \n"]},{"cell_type":"code","execution_count":24,"metadata":{"id":"Mo_5bFeZZVsZ"},"outputs":[],"source":["del all_edges"]},{"cell_type":"code","execution_count":25,"metadata":{"id":"ATHQrp-W6oO4"},"outputs":[],"source":["elts = ug.ep['elts']\n","efts = ug.ep['efts']\n","eclick = ug.ep['eclick']\n","ecart = ug.ep['ecart']\n","eorder = ug.ep['eorder']\n","esub = ug.ep['esub']\n"]},{"cell_type":"code","execution_count":26,"metadata":{"id":"YZjodYD826Ms"},"outputs":[],"source":["vp = {}\n","vp['num_sub'] = ug.new_vp('double',  all_vertices[:, -2] ).get_array()\n","vp['consistency'] = ug.new_vp('double',  1 / (1 + np.log(all_vertices[:, -1] + 1)) ).get_array()\n","vp['num_actions'] = ug.new_vp('unsigned int', all_vertices[:, -3]).get_array()"]},{"cell_type":"code","execution_count":27,"metadata":{"id":"8kwuGZgiIsHZ"},"outputs":[],"source":["del all_vertices"]},{"cell_type":"code","execution_count":28,"metadata":{},"outputs":[],"source":["from graph_tool.stats import remove_parallel_edges\n","remove_parallel_edges(ug)"]},{"cell_type":"markdown","metadata":{"id":"o-jT6bO1pcOF","notebookRunGroups":{"groupValue":"1"}},"source":["# Count actions"]},{"cell_type":"code","execution_count":94,"metadata":{"id":"_g_l1D-QpeqQ"},"outputs":[],"source":["from graph_tool import incident_edges_op\n","vp['num_clicks'] = incident_edges_op(ug, \"out\", \"sum\", eclick).get_array()\n","vp['num_carts'] = incident_edges_op(ug, \"out\", \"sum\", ecart).get_array()\n","vp['num_orders'] = incident_edges_op(ug, \"out\", \"sum\", eorder).get_array()"]},{"attachments":{},"cell_type":"markdown","metadata":{"id":"o-jT6bO1pcOF","notebookRunGroups":{"groupValue":"1"}},"source":["# Get degree"]},{"cell_type":"code","execution_count":95,"metadata":{},"outputs":[],"source":["vp['degree'] = ug.get_total_degrees(np.arange(ug.num_vertices()))"]},{"cell_type":"markdown","metadata":{"id":"XNTi2k_UgUKa"},"source":["# Pagerank"]},{"cell_type":"code","execution_count":96,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":210439,"status":"ok","timestamp":1672107303154,"user":{"displayName":"Hải Minh Nguyễn","userId":"16371132940967469863"},"user_tz":-420},"id":"416kIUGtgXCy","outputId":"3eaee454-8535-4a4f-9740-e5acef966dfb"},"outputs":[{"name":"stdout","output_type":"stream","text":["CPU times: user 14min 36s, sys: 413 ms, total: 14min 36s\n","Wall time: 54.2 s\n"]}],"source":["%%time\n","import graph_tool.centrality as gt\n","vp['pr'] = gt.pagerank(ug).get_array()"]},{"attachments":{},"cell_type":"markdown","metadata":{"id":"XNTi2k_UgUKa"},"source":["# Recent 7 day graph features"]},{"cell_type":"code","execution_count":97,"metadata":{},"outputs":[],"source":["#remove item not interact in recent 7 days\n","efilter = (MAX_TS - elts.get_array() - 3600 * 24 * 7) < 0"]},{"cell_type":"code","execution_count":98,"metadata":{},"outputs":[],"source":["from graph_tool import GraphView\n","fug = GraphView(ug, efilt = efilter)"]},{"cell_type":"code","execution_count":99,"metadata":{},"outputs":[],"source":["vp['recent_num_clicks'] = incident_edges_op(fug, \"out\", \"sum\", eclick).get_array()\n","vp['recent_num_carts'] = incident_edges_op(fug, \"out\", \"sum\", ecart).get_array()\n","vp['recent_num_orders'] = incident_edges_op(fug, \"out\", \"sum\", eorder).get_array()\n","vp['recent_degree'] = fug.get_total_degrees(np.arange(fug.num_vertices()))\n","vp['recent_pr'] = gt.pagerank(fug).get_array()"]},{"attachments":{},"cell_type":"markdown","metadata":{"id":"XNTi2k_UgUKa"},"source":["# Item Features"]},{"cell_type":"code","execution_count":100,"metadata":{},"outputs":[{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>aid</th>\n","      <th>recent_day14_type0</th>\n","      <th>recent_day14_type1</th>\n","      <th>recent_day14_type2</th>\n","      <th>recent_day13_type0</th>\n","      <th>recent_day13_type1</th>\n","      <th>recent_day13_type2</th>\n","      <th>recent_day12_type0</th>\n","      <th>recent_day12_type1</th>\n","      <th>recent_day12_type2</th>\n","      <th>...</th>\n","      <th>recent_day4_type2</th>\n","      <th>recent_day3_type0</th>\n","      <th>recent_day3_type1</th>\n","      <th>recent_day3_type2</th>\n","      <th>recent_day2_type0</th>\n","      <th>recent_day2_type1</th>\n","      <th>recent_day2_type2</th>\n","      <th>recent_day1_type0</th>\n","      <th>recent_day1_type1</th>\n","      <th>recent_day1_type2</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>0</td>\n","      <td>3.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>1.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>3.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>1.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>1.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>1.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>1</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>2.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>2.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>2</td>\n","      <td>1.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>1.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>1.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>3</td>\n","      <td>171.0</td>\n","      <td>14.0</td>\n","      <td>4.0</td>\n","      <td>147.0</td>\n","      <td>9.0</td>\n","      <td>0.0</td>\n","      <td>158.0</td>\n","      <td>12.0</td>\n","      <td>3.0</td>\n","      <td>...</td>\n","      <td>1.0</td>\n","      <td>42.0</td>\n","      <td>5.0</td>\n","      <td>0.0</td>\n","      <td>61.0</td>\n","      <td>3.0</td>\n","      <td>0.0</td>\n","      <td>87.0</td>\n","      <td>9.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>4</td>\n","      <td>3.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>7.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>5.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>2.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>3.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>...</th>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","    </tr>\n","    <tr>\n","      <th>1844279</th>\n","      <td>1855598</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>1844280</th>\n","      <td>1855599</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>1.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>1.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>1844281</th>\n","      <td>1855600</td>\n","      <td>6.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>3.0</td>\n","      <td>1.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>1844282</th>\n","      <td>1855601</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","    <tr>\n","      <th>1844283</th>\n","      <td>1855602</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>1.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>...</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","      <td>2.0</td>\n","      <td>0.0</td>\n","      <td>0.0</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>1844284 rows × 43 columns</p>\n","</div>"],"text/plain":["             aid  recent_day14_type0  recent_day14_type1  recent_day14_type2  \\\n","0              0                 3.0                 0.0                 0.0   \n","1              1                 0.0                 0.0                 0.0   \n","2              2                 1.0                 0.0                 0.0   \n","3              3               171.0                14.0                 4.0   \n","4              4                 3.0                 0.0                 0.0   \n","...          ...                 ...                 ...                 ...   \n","1844279  1855598                 0.0                 0.0                 0.0   \n","1844280  1855599                 0.0                 0.0                 0.0   \n","1844281  1855600                 6.0                 0.0                 0.0   \n","1844282  1855601                 0.0                 0.0                 0.0   \n","1844283  1855602                 0.0                 0.0                 0.0   \n","\n","         recent_day13_type0  recent_day13_type1  recent_day13_type2  \\\n","0                       1.0                 0.0                 0.0   \n","1                       2.0                 0.0                 0.0   \n","2                       0.0                 0.0                 0.0   \n","3                     147.0                 9.0                 0.0   \n","4                       7.0                 0.0                 0.0   \n","...                     ...                 ...                 ...   \n","1844279                 0.0                 0.0                 0.0   \n","1844280                 1.0                 0.0                 0.0   \n","1844281                 3.0                 1.0                 0.0   \n","1844282                 0.0                 0.0                 0.0   \n","1844283                 1.0                 0.0                 0.0   \n","\n","         recent_day12_type0  recent_day12_type1  recent_day12_type2  ...  \\\n","0                       3.0                 0.0                 0.0  ...   \n","1                       2.0                 0.0                 0.0  ...   \n","2                       1.0                 0.0                 0.0  ...   \n","3                     158.0                12.0                 3.0  ...   \n","4                       5.0                 0.0                 0.0  ...   \n","...                     ...                 ...                 ...  ...   \n","1844279                 0.0                 0.0                 0.0  ...   \n","1844280                 1.0                 0.0                 0.0  ...   \n","1844281                 0.0                 0.0                 0.0  ...   \n","1844282                 0.0                 0.0                 0.0  ...   \n","1844283                 0.0                 0.0                 0.0  ...   \n","\n","         recent_day4_type2  recent_day3_type0  recent_day3_type1  \\\n","0                      0.0                1.0                0.0   \n","1                      0.0                0.0                0.0   \n","2                      0.0                0.0                0.0   \n","3                      1.0               42.0                5.0   \n","4                      0.0                0.0                0.0   \n","...                    ...                ...                ...   \n","1844279                0.0                0.0                0.0   \n","1844280                0.0                0.0                0.0   \n","1844281                0.0                0.0                0.0   \n","1844282                0.0                0.0                0.0   \n","1844283                0.0                0.0                0.0   \n","\n","         recent_day3_type2  recent_day2_type0  recent_day2_type1  \\\n","0                      0.0                1.0                0.0   \n","1                      0.0                0.0                0.0   \n","2                      0.0                1.0                0.0   \n","3                      0.0               61.0                3.0   \n","4                      0.0                2.0                0.0   \n","...                    ...                ...                ...   \n","1844279                0.0                0.0                0.0   \n","1844280                0.0                0.0                0.0   \n","1844281                0.0                0.0                0.0   \n","1844282                0.0                0.0                0.0   \n","1844283                0.0                0.0                0.0   \n","\n","         recent_day2_type2  recent_day1_type0  recent_day1_type1  \\\n","0                      0.0                1.0                0.0   \n","1                      0.0                0.0                0.0   \n","2                      0.0                0.0                0.0   \n","3                      0.0               87.0                9.0   \n","4                      0.0                3.0                0.0   \n","...                    ...                ...                ...   \n","1844279                0.0                0.0                0.0   \n","1844280                0.0                0.0                0.0   \n","1844281                0.0                0.0                0.0   \n","1844282                0.0                0.0                0.0   \n","1844283                0.0                2.0                0.0   \n","\n","         recent_day1_type2  \n","0                      0.0  \n","1                      0.0  \n","2                      0.0  \n","3                      0.0  \n","4                      0.0  \n","...                    ...  \n","1844279                0.0  \n","1844280                0.0  \n","1844281                0.0  \n","1844282                0.0  \n","1844283                0.0  \n","\n","[1844284 rows x 43 columns]"]},"execution_count":100,"metadata":{},"output_type":"execute_result"}],"source":["item_agg"]},{"cell_type":"code","execution_count":101,"metadata":{},"outputs":[],"source":["item_index = item_agg.aid.values + MAX_SESSION"]},{"cell_type":"code","execution_count":102,"metadata":{},"outputs":[],"source":["for i, col in enumerate(item_agg.columns[1:]):\n","    val = np.zeros(ug.num_vertices(), dtype= int)\n","    val[item_index] = item_agg.values[:, 1 + i]\n","    vp[col] = val "]},{"cell_type":"code","execution_count":103,"metadata":{},"outputs":[],"source":["del item_agg, item_index"]},{"attachments":{},"cell_type":"markdown","metadata":{"id":"XNTi2k_UgUKa"},"source":["# Item Glob Features"]},{"cell_type":"code","execution_count":104,"metadata":{},"outputs":[{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>('aid', '')</th>\n","      <th>('scaled_ts', 'max')</th>\n","      <th>('scaled_ts', 'min')</th>\n","      <th>('time_decay', 'sum')</th>\n","      <th>('click', 'sum')</th>\n","      <th>('cart', 'sum')</th>\n","      <th>('order', 'sum')</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>0</td>\n","      <td>20.883843</td>\n","      <td>0.467789</td>\n","      <td>12.490385</td>\n","      <td>12.490385</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>1</td>\n","      <td>16.956713</td>\n","      <td>4.798958</td>\n","      <td>13.291816</td>\n","      <td>12.986535</td>\n","      <td>0.305281</td>\n","      <td>0.000000</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>2</td>\n","      <td>20.550625</td>\n","      <td>0.435613</td>\n","      <td>6.384989</td>\n","      <td>6.384989</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>3</td>\n","      <td>20.996204</td>\n","      <td>0.555590</td>\n","      <td>857.524007</td>\n","      <td>800.238522</td>\n","      <td>46.531163</td>\n","      <td>10.754322</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>4</td>\n","      <td>20.851551</td>\n","      <td>0.684074</td>\n","      <td>63.791003</td>\n","      <td>60.633490</td>\n","      <td>3.157513</td>\n","      <td>0.000000</td>\n","    </tr>\n","    <tr>\n","      <th>...</th>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","    </tr>\n","    <tr>\n","      <th>1825494</th>\n","      <td>1855598</td>\n","      <td>19.823125</td>\n","      <td>6.447859</td>\n","      <td>2.954250</td>\n","      <td>2.954250</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","    </tr>\n","    <tr>\n","      <th>1825495</th>\n","      <td>1855599</td>\n","      <td>20.061644</td>\n","      <td>4.376366</td>\n","      <td>4.177124</td>\n","      <td>4.177124</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","    </tr>\n","    <tr>\n","      <th>1825496</th>\n","      <td>1855600</td>\n","      <td>20.693553</td>\n","      <td>0.527882</td>\n","      <td>29.381906</td>\n","      <td>28.243843</td>\n","      <td>0.870351</td>\n","      <td>0.267711</td>\n","    </tr>\n","    <tr>\n","      <th>1825497</th>\n","      <td>1855601</td>\n","      <td>19.285602</td>\n","      <td>0.243924</td>\n","      <td>23.012059</td>\n","      <td>20.653915</td>\n","      <td>2.358144</td>\n","      <td>0.000000</td>\n","    </tr>\n","    <tr>\n","      <th>1825498</th>\n","      <td>1855602</td>\n","      <td>19.961296</td>\n","      <td>2.324711</td>\n","      <td>5.139038</td>\n","      <td>5.139038</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>1825499 rows × 7 columns</p>\n","</div>"],"text/plain":["         ('aid', '')  ('scaled_ts', 'max')  ('scaled_ts', 'min')  \\\n","0                  0             20.883843              0.467789   \n","1                  1             16.956713              4.798958   \n","2                  2             20.550625              0.435613   \n","3                  3             20.996204              0.555590   \n","4                  4             20.851551              0.684074   \n","...              ...                   ...                   ...   \n","1825494      1855598             19.823125              6.447859   \n","1825495      1855599             20.061644              4.376366   \n","1825496      1855600             20.693553              0.527882   \n","1825497      1855601             19.285602              0.243924   \n","1825498      1855602             19.961296              2.324711   \n","\n","         ('time_decay', 'sum')  ('click', 'sum')  ('cart', 'sum')  \\\n","0                    12.490385         12.490385         0.000000   \n","1                    13.291816         12.986535         0.305281   \n","2                     6.384989          6.384989         0.000000   \n","3                   857.524007        800.238522        46.531163   \n","4                    63.791003         60.633490         3.157513   \n","...                        ...               ...              ...   \n","1825494               2.954250          2.954250         0.000000   \n","1825495               4.177124          4.177124         0.000000   \n","1825496              29.381906         28.243843         0.870351   \n","1825497              23.012059         20.653915         2.358144   \n","1825498               5.139038          5.139038         0.000000   \n","\n","         ('order', 'sum')  \n","0                0.000000  \n","1                0.000000  \n","2                0.000000  \n","3               10.754322  \n","4                0.000000  \n","...                   ...  \n","1825494          0.000000  \n","1825495          0.000000  \n","1825496          0.267711  \n","1825497          0.000000  \n","1825498          0.000000  \n","\n","[1825499 rows x 7 columns]"]},"execution_count":104,"metadata":{},"output_type":"execute_result"}],"source":["item_glob"]},{"cell_type":"code","execution_count":105,"metadata":{},"outputs":[],"source":["item_index = item_glob.iloc[:,0].values + MAX_SESSION"]},{"cell_type":"code","execution_count":106,"metadata":{},"outputs":[],"source":["from meow_utils import glob_features"]},{"cell_type":"code","execution_count":107,"metadata":{},"outputs":[],"source":["for i, col in enumerate(glob_features[:-1]):\n","    val = np.zeros(ug.num_vertices(), dtype= float)\n","    val[item_index] = item_glob.values[:, 1 + i]\n","    vp[col] = val \n"]},{"cell_type":"code","execution_count":108,"metadata":{},"outputs":[],"source":["vp['item_glob_durability']= np.zeros(ug.num_vertices(), dtype= float)\n","vp['item_glob_durability'][item_index] = item_glob.iloc[:,1].values - item_glob.iloc[:,2].values\n"]},{"cell_type":"code","execution_count":109,"metadata":{},"outputs":[],"source":["del item_glob, item_index\n"]},{"cell_type":"markdown","metadata":{"id":"iUY4DIJiGsio"},"source":["# Create Disk VP"]},{"cell_type":"code","execution_count":110,"metadata":{},"outputs":[{"data":{"text/plain":["dict_keys(['num_sub', 'consistency', 'num_actions', 'num_clicks', 'num_carts', 'num_orders', 'degree', 'pr', 'recent_num_clicks', 'recent_num_carts', 'recent_num_orders', 'recent_degree', 'recent_pr', 'recent_day14_type0', 'recent_day14_type1', 'recent_day14_type2', 'recent_day13_type0', 'recent_day13_type1', 'recent_day13_type2', 'recent_day12_type0', 'recent_day12_type1', 'recent_day12_type2', 'recent_day11_type0', 'recent_day11_type1', 'recent_day11_type2', 'recent_day10_type0', 'recent_day10_type1', 'recent_day10_type2', 'recent_day9_type0', 'recent_day9_type1', 'recent_day9_type2', 'recent_day8_type0', 'recent_day8_type1', 'recent_day8_type2', 'recent_day7_type0', 'recent_day7_type1', 'recent_day7_type2', 'recent_day6_type0', 'recent_day6_type1', 'recent_day6_type2', 'recent_day5_type0', 'recent_day5_type1', 'recent_day5_type2', 'recent_day4_type0', 'recent_day4_type1', 'recent_day4_type2', 'recent_day3_type0', 'recent_day3_type1', 'recent_day3_type2', 'recent_day2_type0', 'recent_day2_type1', 'recent_day2_type2', 'recent_day1_type0', 'recent_day1_type1', 'recent_day1_type2', 'item_glob_last_action', 'item_glob_first_action', 'item_glob_time_decay_sum', 'item_glob_time_decay_sum_click', 'item_glob_time_decay_sum_cart', 'item_glob_time_decay_sum_order', 'item_glob_durability'])"]},"execution_count":110,"metadata":{},"output_type":"execute_result"}],"source":["vp.keys()"]},{"cell_type":"code","execution_count":111,"metadata":{"id":"ITFlt6yqGwVV"},"outputs":[],"source":["shape = None\n","for k in vp.keys():\n","  vp[k] = np.array(vp[k])\n","  if shape:\n","    assert vp[k].shape == shape, (k, vp[k].shape, shape)\n","  else:\n","    shape = vp[k].shape"]},{"cell_type":"code","execution_count":112,"metadata":{"id":"DygoJuUGc-2q"},"outputs":[{"data":{"text/plain":["(62, 14755382)"]},"execution_count":112,"metadata":{},"output_type":"execute_result"}],"source":["import multiprocessing\n","mem_size = (len(vp.keys()), vp[k].shape[0])\n","mem_size"]},{"cell_type":"code","execution_count":113,"metadata":{},"outputs":[],"source":["disk_vp = np.memmap('splitted_vertice_properties.array', dtype='float64', mode='w+', shape=(mem_size[0],mem_size[1]))\n"]},{"cell_type":"code","execution_count":48,"metadata":{},"outputs":[],"source":["disk_vp = np.memmap('vertice_properties.array', dtype='float64', mode='w+', shape=(mem_size[0],mem_size[1]))\n"]},{"cell_type":"code","execution_count":114,"metadata":{"id":"UEOTGNxAd9Oo"},"outputs":[],"source":["for i, k in enumerate(vp.keys()):\n","  disk_vp[i] = vp[k] \n"]},{"cell_type":"code","execution_count":115,"metadata":{"id":"kALTvW8eeFd9"},"outputs":[],"source":["del vp"]},{"attachments":{},"cell_type":"markdown","metadata":{"id":"iUY4DIJiGsio"},"source":["# Load VP"]},{"cell_type":"code","execution_count":29,"metadata":{},"outputs":[],"source":["assert disk_vp.shape[1] == ug.num_vertices(), (disk_vp.shape[1], ug.num_vertices())"]},{"cell_type":"markdown","metadata":{"id":"VelelmsEMYdG"},"source":["# Load test"]},{"cell_type":"code","execution_count":29,"metadata":{"executionInfo":{"elapsed":420,"status":"ok","timestamp":1672118239002,"user":{"displayName":"Hải Minh Nguyễn","userId":"16371132940967469863"},"user_tz":-420},"id":"yZ0VUDI3m_zV"},"outputs":[],"source":["ROOT = '/home/anhphantq/otto'\n","DATA_FOLDER = ROOT + '/data'\n","\n"]},{"cell_type":"code","execution_count":30,"metadata":{"executionInfo":{"elapsed":10,"status":"ok","timestamp":1672118239518,"user":{"displayName":"Hải Minh Nguyễn","userId":"16371132940967469863"},"user_tz":-420},"id":"T6hunU2XM7Nt"},"outputs":[],"source":["ROOT = '/home/anhphantq/otto'\n","DATA_FOLDER = ROOT + '/splitted_data'\n","\n"]},{"cell_type":"code","execution_count":31,"metadata":{"executionInfo":{"elapsed":9,"status":"ok","timestamp":1672118239519,"user":{"displayName":"Hải Minh Nguyễn","userId":"16371132940967469863"},"user_tz":-420},"id":"xnWR1DPHMjRr"},"outputs":[],"source":["import pandas as pd\n","test_df = pd.read_parquet(DATA_FOLDER + '/test.parquet')"]},{"cell_type":"code","execution_count":32,"metadata":{"executionInfo":{"elapsed":310,"status":"ok","timestamp":1672118239821,"user":{"displayName":"Hải Minh Nguyễn","userId":"16371132940967469863"},"user_tz":-420},"id":"TSGA3l7rnJSP"},"outputs":[],"source":["import numpy as np\n","session_list = np.sort(test_df['session'].unique())\n"]},{"cell_type":"code","execution_count":33,"metadata":{"executionInfo":{"elapsed":6,"status":"ok","timestamp":1672118239821,"user":{"displayName":"Hải Minh Nguyễn","userId":"16371132940967469863"},"user_tz":-420},"id":"Yi41g4u0nNwe"},"outputs":[],"source":["session_list = session_list[:len(session_list) // 20]"]},{"cell_type":"code","execution_count":120,"metadata":{"executionInfo":{"elapsed":6,"status":"ok","timestamp":1672118239822,"user":{"displayName":"Hải Minh Nguyễn","userId":"16371132940967469863"},"user_tz":-420},"id":"qfw4a8T0nSPn"},"outputs":[],"source":["session_list = session_list[1 * len(session_list) // 20 : 15 * len(session_list) // 20]\n"]},{"cell_type":"code","execution_count":34,"metadata":{},"outputs":[{"data":{"text/plain":["90062"]},"execution_count":34,"metadata":{},"output_type":"execute_result"}],"source":["len(session_list)"]},{"cell_type":"code","execution_count":35,"metadata":{"executionInfo":{"elapsed":6,"status":"ok","timestamp":1672118239823,"user":{"displayName":"Hải Minh Nguyễn","userId":"16371132940967469863"},"user_tz":-420},"id":"VoVsduZepRLg"},"outputs":[],"source":["del test_df"]},{"cell_type":"markdown","metadata":{"id":"tgE2wrB_54QH"},"source":["# Find neighbour strategy"]},{"cell_type":"code","execution_count":36,"metadata":{},"outputs":[{"data":{"text/plain":["(7, 59)"]},"execution_count":36,"metadata":{},"output_type":"execute_result"}],"source":["from meow_utils import USER_FEATURES, ITEM_FEATURES, \\\n","feature_id_map, recent_features_id_map, recent_features, \\\n","glob_features_id_map, glob_features\n","\n","user_feature_ids = [feature_id_map[f] for f in USER_FEATURES]\n","all_item_feature_ids = [feature_id_map[f] for f in ITEM_FEATURES]\n","\n","recent_feature_ids = [recent_features_id_map[f] for f in recent_features]\n","glob_feature_ids = [glob_features_id_map[f] for f in glob_features]\n","len(user_feature_ids), len(all_item_feature_ids)"]},{"cell_type":"code","execution_count":37,"metadata":{"id":"DWmDgIEc57U9"},"outputs":[],"source":["\n","def get_user_features(user):\n","  return [disk_vp[feature_id_map[f]][user] for f in USER_FEATURES]\n","\n","def get_item_features(items, id_list):\n","\n","  return [\n","      np.array([\n","          disk_vp[i][item] for item in items \n","      ]) for i in id_list\n","  ]\n","\n","\n","def get_test_items(user, beta):\n","  global ug, fug, elts, eclick, ecart, eorder, get_test_items_params, FILTER\n","  #0: src, 1: target, 2: lts, 3: eclick, 4: ecart, 5:eorder 6 subsession, 7 ftss\n","  edges = ug.get_all_edges(user, eprops = [elts, eclick, ecart, eorder, esub, efts])\n","        \n","  return _get_test_items(edges, beta, **get_test_items_params)\n","\n","\n","def get_user_neighbours(item):\n","  global ug, fug, elts, efts, eclick, ecart, eorder, esub, FILTER\n","  #1 target 2 ltss 3 ftss 4 click 5 cart 6 order 7 subsession\n","  \n","  \n","  edges = ug.get_all_edges(item, eprops = [elts, efts, eclick, ecart, eorder, esub])\n","  \n","  reverse_num_people_interacts = 1  / (1 + np.log(edges.shape[0] + 1))\n","  \n","  reverse_num_actions = 1  / (1 + np.log(edges[:,4].sum() + edges[:,5].sum() + edges[:,6].sum() + 1))\n","  \n","  reverse_item_appear_prob = reverse_num_people_interacts * reverse_num_actions \n","  return edges[:,1], edges[:,2], edges[:,3], edges[:, 4], edges[:, 5], edges[:, 6], edges[:, 7], reverse_item_appear_prob\n","  \n","\n","def get_item_neighbours(user, lts, fts, prevsub, prev_click, prev_cart, prev_order, alpha):\n","  global ug, fug, elts, efts, eclick, ecart, eorder, esub, get_items_params, FILTER\n","  #edge idx: 0 is src, 1 is target, 2 is last time stamp, 3 is first time stamp, 4 click, 5 cart, 6 order, 7 subsession\n","\n","  edges = ug.get_all_edges(user, eprops = [elts, efts, eclick, ecart, eorder, esub])\n","  assert len(edges) == len(np.unique(edges[:, 1])), (len(edges) , len(np.unique(edges[:, 1])))\n","  return _get_item_neighbours(edges, lts, fts, prev_click = prev_click, prev_cart = prev_cart, prev_order = prev_order\n","                              , alpha = alpha, prevsub = prevsub, **get_items_params)\n"]},{"cell_type":"markdown","metadata":{"id":"IA7l7bDZlSTU"},"source":["# Numba funcs"]},{"cell_type":"code","execution_count":38,"metadata":{"id":"ZdF8Yu6L6hh_"},"outputs":[],"source":["import numba as nb\n","import numpy as np\n","from numba import prange\n","    \n","@nb.njit([nb.float64[:](nb.int64[:]), nb.float64[:](nb.float64[:])], cache = True, nogil = True)\n","def reverse_normalize(X):\n","  m = np.max(X)\n","  return (m - X) / (m - np.min(X) + 1e-12) \n","\n","@nb.njit(nb.float64[:](nb.int64[:]), cache = True, nogil = True)\n","def normalize(X):\n","  m = np.max(X)\n","  return (X - np.min(X)) / (m - np.min(X) + 1e-12) \n","\n","@nb.njit([nb.int64[:](nb.float64[:]),\n","          nb.int64[:](nb.int64[:]),\n","          ], cache = True, nogil = True)\n","def get_rank(x):\n","  return np.argsort(np.argsort(x))\n","\n","@nb.njit(nb.int64(nb.int64), cache = True, nogil = True)\n","def decode_1d(x):\n","  rs = 0\n","  while x > 0:\n","    if x % 2 == 1:\n","      rs += 1\n","    x = x // 2\n","  return rs\n","\n","@nb.njit(nb.int64[:](nb.int64[:]), cache = True, nogil = True)\n","def decode_2d(x):\n","  return np.array([decode_1d(i) for i in x])\n","\n","\n","\n","@nb.njit([\n","  nb.float64[:, :](nb.float64[:,:]),\n","  nb.int64[:, :](nb.int64[:,:]),\n","          ])\n","def cumsum_2d(x):\n","  result = np.empty_like(x)\n","  result[:, 0] = x[:, 0]\n","  for i in prange(1, x.shape[1]):\n","    result[:, i] = x[:, i] +  result[:, i - 1]\n","  return result\n","\n","\n","@nb.njit([nb.float64[:](nb.float64[:], nb.float64, nb.float64),\n","          nb.float64[:](nb.float64[:], nb.int64, nb.float64),\n","          nb.float64[:](nb.float64[:], nb.float64, nb.int64),\n","          nb.float64[:](nb.int64[:], nb.float64, nb.int64),\n","         nb.float64[:](nb.float64[:], nb.int64, nb.int64)], cache = True, nogil = True)\n","def normalize_to_range(X, l, r):\n","  m = np.min(X)\n","  return (X - m) / (np.max(X) - m + 1e-12) * (r -l) + l\n","\n","\n","@nb.njit([\n","          nb.types.Tuple((nb.int64[:], nb.float64[:,:], nb.int64[:])) (nb.int64[:], nb.float64[:,:]),\n","          nb.types.Tuple((nb.int64[:], nb.int64[:,:], nb.int64[:])) (nb.int64[:], nb.int64[:,:]),\n","          ], cache = True, nogil = True)\n","def sum_by_group_multiple_values_return_num_appearance(groups, values):\n","    #values shape n_values * n_items\n","    order = np.argsort(groups)\n","\n","    groups = groups[order]\n","    values = values[:, order]\n","    values = cumsum_2d(values)\n","\n","    index = np.ones(len(groups), 'bool')\n","    index[:-1] = groups[1:] != groups[:-1]\n","    nodes = np.hstack((np.array([0]), np.where(index)[0] + 1))\n","\n","    values = values[:, index]\n","\n","    groups = groups[index]\n","    values[:, 1:] = values[:, 1:] - values[:, :-1]\n","    return groups, values, nodes[1:] - nodes[:-1]\n","\n","@nb.njit([\n","          nb.types.Tuple((nb.int64[:], nb.float64[:,:])) (nb.int64[:], nb.float64[:,:]),\n","          nb.types.Tuple((nb.int64[:], nb.int64[:,:])) (nb.int64[:], nb.int64[:,:]),\n","          ], cache = True, nogil = True)\n","def sum_by_group_multiple_values(groups, values):\n","    #values shape n_values * n_items\n","    order = np.argsort(groups)\n","\n","    groups = groups[order]\n","    values = values[:, order]\n","    values = cumsum_2d(values)\n","\n","    index = np.ones(len(groups), 'bool')\n","    index[:-1] = groups[1:] != groups[:-1]\n","    \n","\n","    values = values[:, index]\n","\n","    groups = groups[index]\n","    values[:, 1:] = values[:, 1:] - values[:, :-1]\n","    return groups, values\n","\n","@nb.njit([\n","          nb.types.Tuple((nb.int64[:], nb.float64[:, :], nb.int64[:])) (nb.int64[:], nb.float64[:,:], nb.int64[:]),\n","          nb.types.Tuple((nb.int64[:], nb.int64[:, :], nb.int64[:])) (nb.int64[:], nb.int64[:,:], nb.int64[:]),\n","          ], cache = True, nogil = True)\n","def sum_by_group_exclude_multiple_values_return_num_appearance(groups, values, exclude):\n","    #remove existed items\n","    inclusive = np.ones(len(groups), 'bool')\n","    for e in exclude:\n","      inclusive = inclusive & (groups != e)\n","    #values shape n_values * n_items\n","    return  sum_by_group_multiple_values_return_num_appearance(groups[inclusive], values[:, inclusive])\n","\n","    \n","@nb.njit([\n","          nb.types.Tuple((nb.int64[:], nb.float64[:,:,:])) (nb.int64[:], nb.float64[:,:,:]),\n","          ], cache = True, nogil = True)\n","def sum_by_group_multiple_values_2d(groups, values):\n","    order = np.argsort(groups)\n","\n","    groups = groups[order]\n","    #first axis of values  is 3 types of actions , second axisis heterogenous, third axis is different items\n","    values = values[:, :, order]\n","    for i in prange(values.shape[0]):\n","      values[i] = cumsum_2d(values[i])\n","\n","    index = np.ones(len(groups), 'bool')\n","    index[:-1] = groups[1:] != groups[:-1]\n","\n","    values = values[:, :, index]\n","\n","    groups = groups[index]\n","    values[:, :, 1:] = values[:, :,1:] - values[:, :,:-1]\n","\n","    return groups, values\n","\n","@nb.njit([\n","          nb.types.Tuple((nb.int64[:], nb.int64[:,:])) (nb.int64[:], nb.int64[:,:]),\n","          nb.types.Tuple((nb.int64[:], nb.float64[:,:])) (nb.int64[:], nb.float64[:,:]),\n","          ], cache = True, nogil = True)\n","def max_by_group_multiple_values(groups, values):\n","    order = np.argsort(groups)\n","\n","    groups = groups[order]\n","    #first axis of values  is 3 types of actions , second axisis heterogenous, third axis is different items\n","    values = values[ :, order]\n","    \n","\n","    index = np.ones(len(groups), 'bool')\n","    index[:-1] = groups[1:] != groups[:-1]\n","    groups = groups[index]\n","    ends = np.where(index)[0]\n","\n","    k = 0\n","    for i in prange(1, values.shape[1]):\n","        if i != ends[k] + 1:\n","            values[:, i] = np.maximum(values[:, i], values[:, i - 1])\n","        else:\n","            k+=1\n","    \n","    values = values[:, index]\n","\n","    return groups, values\n","          \n","\n","@nb.njit([\n","          nb.types.Tuple((nb.int64[:], nb.float64[:], nb.int64[:], nb.boolean[:])) (nb.int64[:], nb.float64[:], nb.int64, nb.float64),\n","          nb.types.Tuple((nb.int64[:], nb.int64[:], nb.int64[:], nb.boolean[:])) (nb.int64[:], nb.int64[:], nb.int64, nb.float64),\n","          # nb.types.Tuple((nb.int64[:], nb.int64[:], nb.int64[:], nb.boolean[:])) (nb.float64[:], nb.float64[:], nb.int64, nb.float64),\n","          ], cache = True, nogil = True)\n","def find_topk_user(cousers, cofitness, min_topk_user, cofitness_thres):\n","  #sort cousers ascending by id\n","  order = np.argsort(cousers)\n","  cousers = cousers[order]\n","  cofitness = cofitness[order]\n","  \n","\n","  #calculate groupby sum of cousers\n","  cumcofitness = np.cumsum(cofitness)\n","\n","  #find unique\n","  index = np.ones(len(cousers), 'bool')\n","\n","  #this is the last index of each group\n","  index[:-1] = cousers[1:] != cousers[:-1]\n","  cumcofitness = cumcofitness[index]\n","  \n","\n","  #find start and end of groups\n","  nodes = np.hstack((np.array([0]), np.where(index)[0] + 1))\n","\n","  starts = nodes[:-1]\n","  ends = nodes[1:]\n","  lens = ends - starts\n","  rank_nums = get_rank(lens)\n","    \n","  #caluculate groupby sum of cousers\n","  cumcofitness[1:] = cumcofitness[1:] - cumcofitness[:-1]\n","\n","  rank_fit = get_rank(cumcofitness)\n","  \n","  order_fit = np.argsort(-(rank_nums + rank_fit))\n","\n","  k = min_topk_user\n","  if cofitness_thres > 0:\n","    #get topk user\n","    #dynamic top user\n","    total_fitness = np.sum(cumcofitness)\n","    \n","    k = 0\n","    remain = 0\n","    thresh_hold = cofitness_thres * total_fitness\n","    while remain < thresh_hold:\n","      remain += cumcofitness[order_fit][k]\n","      k+=1 \n","    \n","    if k < min_topk_user:\n","      k = min_topk_user\n","\n","    if k > 500:\n","      k = 500\n","\n","  \n","  order_fit = order_fit[:k]\n","\n","  starts = nodes[order_fit]\n","  ends = nodes[order_fit + 1]\n","  \n","  filter = np.zeros(len(cousers), 'bool')\n","  for i in prange(len(starts)):\n","      filter[starts[i] : ends[i]] = 1\n","      num_cointeract = ends[i] - starts[i]\n","      score = 1 if num_cointeract < 2 else (6 if num_cointeract < 4 else 10)\n","      cofitness[starts[i] : ends[i]] = cofitness[starts[i] : ends[i]] * score\n","  return cousers[filter], cofitness[filter], order, filter"]},{"cell_type":"code","execution_count":39,"metadata":{},"outputs":[],"source":["@nb.njit(nb.types.Tuple((nb.int64[:], nb.float64[:],\n","                         nb.int64[:], nb.int64[:],\n","                         nb.int64[:], nb.int64[:],\n","                         nb.float64[:], nb.float64[:],\n","                         nb.float64[:], nb.int64[:],\n","                         nb.int64[:]))(nb.int64[:, :], nb.float64, nb.float64, nb.float64, nb.float64, nb.float64, nb.int64) , cache = True, nogil = True)\n","def _get_test_items(edges, beta, test_click_weight, test_cart_weight, test_order_weight_l, test_order_weight_s, day):\n","  #0: src, 1: target, 2: lts, 3: eclick, 4: ecart, 5:eorder 6:susbsession, 7 fts\n","  \n","  \n","  time_decay = beta ** reverse_normalize(edges[:, 2])\n","  \n","  order_weight = np.where(edges[:, 5] > 1, test_order_weight_l, test_order_weight_s)\n","\n","  weight = edges[:, 3] * test_click_weight + edges[:, 4] * test_cart_weight + order_weight \n","\n","  fitness =  time_decay * weight \n","\n","  order = np.argsort(-fitness)\n","\n","  num_subsession = decode_2d(edges[ :, 6])\n","  return edges[order, 1], fitness[order], edges[order, 3] , edges[order, 4], edges[order, 5], \\\n","          num_subsession[order], time_decay[order], normalize_to_range(edges[order,2], 0.3, 1), normalize_to_range(edges[order, 7], 0.3, 1), edges[order, 2] - edges[order, 7], edges[order, 3] + edges[order, 4] + edges[order, 5]"]},{"cell_type":"code","execution_count":40,"metadata":{},"outputs":[],"source":["@nb.njit([\n","          nb.types.Tuple((nb.int64[:], nb.float64[:,:], nb.int64[:]))(nb.int64[:,:], nb.int64, nb.int64, nb.int64, nb.int64, nb.float64, nb.int64, nb.float64, nb.float64, nb.float64 , nb.float64, nb.int64), \n","          nb.types.Tuple((nb.int64[:], nb.float64[:,:], nb.int64[:]))(nb.int64[:,:], nb.int64, nb.int64, nb.int64, nb.int64, nb.int64,   nb.int64, nb.float64, nb.int64, nb.int64, nb.int64, nb.int64), \n","          ], cache = True, nogil = True)\n","def _get_item_neighbours(edges, lts, fts, prevsub, prev_click, prev_cart, prev_order, alpha, click_weight, cart_weight, order_weight, day):\n","  #edge idx: 0 is src, 1 is target, 2 is last time stamp, 3 is first time stamp, 4 click, 5 cart, 6 order, 7 -> subsession\n","  \n","  cosubsession = prevsub & edges[:, 7]\n","  cosubsession = decode_2d(cosubsession)\n","  subsession_coef = 1.8 ** np.clip(cosubsession, 0, 1)\n","\n","  reverse_user_variety = 1  / (1 + np.log(np.sum(decode_2d(edges[:, 7])) + 1))\n","  reverse_user_consistency = 1 / (1 + np.log((np.amax(edges[:,2]) - np.amin(edges[:, 3])) + 1))\n","\n","\n","  coclick = np.sqrt(edges[:, 4] * prev_click)\n","  cocart = np.sqrt(edges[:, 5] * prev_cart)\n","  coorder = np.sqrt(edges[:, 6] * prev_order)\n","  \n","  action_weight = np.array([3, 6, 1])\n","  prev_interactions = np.array([prev_click, prev_cart, prev_order])\n","  cointeractions = np.empty((9, edges.shape[0]))\n","  for i in prange(3):\n","    for j in prange(3):\n","      \n","      cointeractions[i * 3 + j] = np.sqrt(edges[:, i + 4] * prev_interactions[j]) * np.sqrt(action_weight[i] * action_weight[j])\n","      \n","  \n","  #calculate time diff\n","  u1 = np.absolute(lts - edges[:,3])\n","  u2 = np.absolute(fts - edges[:,2])\n","  union = np.maximum(u1, u2)\n","  \n","  \n","  \n","  l1 = lts - fts \n","  l2 = edges[:, 2] - edges[:, 3]\n","\n","  \n","  time_diff = (union - (l1 + l2) / 2) / day \n","\n","  time_decay = alpha ** time_diff\n","  \n","  is_happend_later = np.where(edges[:, 3] > fts, 2, 1)\n","  \n","  history_order = BETA ** reverse_normalize(edges[:, 2])\n","    \n","  scale_coef = is_happend_later * reverse_user_variety * reverse_user_consistency * history_order\n","  \n","  \n","  coclick_sub_coef = coclick  * subsession_coef * scale_coef\n","  coclick_time_decay = coclick * time_decay  * scale_coef\n","  \n","  cocart_sub_coef = cocart  * subsession_coef * scale_coef\n","  cocart_time_decay = cocart * time_decay  * scale_coef\n","  \n","  coorder_sub_coef = coorder  * subsession_coef * scale_coef\n","  coorder_time_decay = coorder * time_decay  * scale_coef\n","  \n","  \n","  \n","  fitness_subco = coclick_sub_coef * click_weight + cocart_sub_coef * cart_weight + coorder_sub_coef * order_weight\n","\n","  fitness_time_decay = coclick_time_decay * click_weight + cocart_time_decay * cart_weight + coorder_time_decay * order_weight\n","  \n","  cointeractions_subco = cointeractions * subsession_coef * scale_coef\n","  cointeractions_time_decay = cointeractions * time_decay * scale_coef\n","  #mark k most recent items\n","  k = 1\n","  if edges.shape[0] <= k:\n","    is_k_most_recent_items =  np.full(edges.shape[0], 1, dtype = np.float64)\n","  else:\n","    is_k_most_recent_items = (edges[:, 2] > edges[np.argsort( -edges[:, 2])[k], 2]).astype(np.float64)\n","  \n","  \n","  num_happend_later = np.where(edges[:, 3] > fts, 1, 0)\n","  num_happend_before = 1 - num_happend_later\n","  \n","  values_to_return = np.stack((\n","      fitness_subco, fitness_time_decay, cosubsession,\n","      coclick_sub_coef, cocart_sub_coef, coorder_sub_coef, coclick_time_decay, cocart_time_decay, coorder_time_decay, is_k_most_recent_items, num_happend_later, num_happend_before\n","  ), axis = 1)\n","  \n","  values_to_return = np.concatenate((values_to_return, np.transpose(cointeractions_subco), np.transpose(cointeractions_time_decay)), axis = 1)\n","  return edges[:, 1], values_to_return, edges[:, 2]"]},{"cell_type":"markdown","metadata":{"id":"N3IpBw4N4jbO"},"source":["# Step"]},{"cell_type":"code","execution_count":41,"metadata":{"executionInfo":{"elapsed":292,"status":"ok","timestamp":1672116821830,"user":{"displayName":"Hải Minh Nguyễn","userId":"16371132940967469863"},"user_tz":-420},"id":"HnQUJmCJBAM3"},"outputs":[],"source":["from meow_utils import create_level1_data, create_level2_data, level1_columns, level2_columns, lincom_weights, index_recent_features\n","\n","def get_type_items(user, data= None, ):\n","\n","  global ALPHA, BETA, MIN_TOPK_COUSER, COFITNESS_THRES, Meow, GET_RAW, top_20_clicks, top_20_by2buy, top_20_buys\n","  #get user features\n","  \n","  ori_user_features = get_user_features(user)\n","  \n","  #fitness sorted descending\n","  items, fitness, num_clicks, num_carts, num_orders, \\\n","   num_subsession, time_decay, lts, fts, consistency, num_interacts = get_test_items(user, BETA)\n","\n","  \n","  #create level 1 features\n","  n = len(items)\n","\n","  level1_features = create_level1_data(np.transpose(np.stack((\n","                                      items[:n],\n","                                      fitness[:n],\n","                                      num_clicks[:n],\n","                                      num_carts[:n],\n","                                      num_orders[:n],\n","                                      num_subsession[:n],\n","                                      time_decay[:n],\n","                                      lts[:n],\n","                                      fts[:n],\n","                                      consistency[:n],\n","                                      num_interacts[:n],\n","                                      np.full(n, 1),\n","                                      ))))\n","      \n","  #start look for level 2 items\n","    \n","  #get couser\n","  coalls = [get_user_neighbours(items[i]) for i in range(len(items))] \n","  \n","  #multiply item fitness with reverse of prob of that item appear\n","  cofitness = np.concatenate([np.full_like(all[0], fitness[i]) * all[7] for i, all in enumerate(coalls)] )\n","  cousers = np.concatenate([all[0] for all in coalls])\n","\n","  #find topk cousers\n","  cousers, cofitness, order, filter  = find_topk_user(cousers, cofitness, MIN_TOPK_COUSER, COFITNESS_THRES)\n","  coltss = np.concatenate([all[1] for all in coalls])[order][filter]\n","  coftss = np.concatenate([all[2] for all in coalls])[order][filter]\n","  coclick = np.concatenate([all[3] for all in coalls])[order][filter]\n","  cocart = np.concatenate([all[4] for all in coalls])[order][filter]\n","  coorder = np.concatenate([all[5] for all in coalls])[order][filter]\n","  cosubsession = np.concatenate([all[6] for all in coalls])[order][filter]\n","  \n","  #get coitems\n","  coalls = [get_item_neighbours(cousers[i], coltss[i], coftss[i], cosubsession[i], coclick[i], cocart[i], coorder[i], ALPHA) for i in range(len(cousers))]\n","  coitems = np.concatenate([all[0] for all in coalls])\n","  \n","\n","  #multiply item by fitness of user which interacted with it\n","  \n","  cofitness = np.transpose(np.concatenate([all[1]* cofitness[i] for i, all in enumerate(coalls)]) )\n","  last_interact = np.concatenate([all[2] for all in coalls]) \n","  _, last_interact = max_by_group_multiple_values(coitems.copy(), last_interact.reshape(1, -1))\n","  #count covisit\n","  coitems, cofitness, num_appearance= sum_by_group_multiple_values_return_num_appearance(coitems, cofitness)\n","  \n","  assert np.ravel(last_interact).shape[0] == np.ravel(coitems).shape[0], (last_interact.shape, coitems.shape)\n","  \n","  \n","  \n","  \n","    \n","  cofitness = cofitness / (1 + np.log(np.maximum(MAX_TS - last_interact, 0) / 3600 / 24 + 1))\n","  cofitness_cosub, cofitness_time_decay, num_cosub,\\\n","  coclick_sub_coef, cocart_sub_coef, coorder_sub_coef,\\\n","  coclick_time_decay, cocart_time_decay, coorder_time_decay, is_k_most_recent_items, num_happend_later, num_happend_before = (cofitness[i] for i in range(cofitness.shape[0] - 18))\n","  \n","  cointeractions_sub_coef = cofitness[-18: -9]\n","  cointeractions_time_decay = cofitness[-9:]\n","  \n","    \n","  happend_later_ratio = num_happend_later / (num_happend_later + num_happend_before + 1e-12)\n","  #get item in covisit\n","  index_in_level1 = np.ravel(np.argwhere(np.in1d(coitems, items)))\n","  \n","  \n","  \n","  rank_cosub = get_rank(cofitness_cosub) \n","  rank_time_decay = get_rank(cofitness_time_decay) \n","  rank_num_appearance = get_rank(num_appearance) * 0.2 \n","  rank_cointeraction_cosub = get_rank(np.sum(cointeractions_sub_coef, axis = 0)) * 20\n","\n","  \n","  rank_cointeractions_time_decay = get_rank(np.sum(cointeractions_time_decay, axis = 0)) * 20\n","\n","  rank_sum = rank_cosub  + rank_time_decay + rank_num_appearance + rank_cointeraction_cosub + rank_cointeractions_time_decay\n","  \n","  \n","  order = np.argsort(-rank_sum)[:MAX_NUM_CANDS]\n","  \n","  \n","  \n","  if GET_RAW:\n","    data.append(items.tolist() + [i for i in coitems[order] if i not in set(items.tolist())])\n","    return\n","  \n","  #get most recent items from users\n","  order_most_recent = np.argsort(-is_k_most_recent_items)[:MAX_NUM_CANDS]\n","  order = np.union1d(order, order_most_recent)\n","  #make sure all item in level 1 appended\n","  order = np.union1d(order, index_in_level1)\n","  if len(coitems):\n","    n = len(order)\n","    need_rerank = True\n","\n","    features = [num_cosub[order],\n","                coclick_sub_coef[order],\n","                cocart_sub_coef[order],\n","                coorder_sub_coef[order],\n","                coclick_time_decay[order],\n","                cocart_time_decay[order],\n","                coorder_time_decay[order],                  \n","                num_appearance[order],\n","                is_k_most_recent_items[order],\n","                num_happend_later[order],\n","                num_happend_before[order],\n","                happend_later_ratio[order],\n","                np.ravel(last_interact)[order],\n","\n","                ]\n","    \n","    lincom_features = []\n","    \n","    for w1, w2, w3 in lincom_weights:\n","      lincom_features.append(coclick_sub_coef[order] * w1 + cocart_sub_coef[order] * w2 + coorder_sub_coef[order] * w3)\n","      lincom_features.append(coclick_time_decay[order] * w1 + cocart_time_decay[order] * w2 + coorder_time_decay[order] * w3)\n","    \n","    \n","    #quotient features\n","    norm_features = []\n","    for f in features + lincom_features:\n","      mean  = f.mean()\n","      norm_features.append(f / (mean + 1e-12))\n","      \n","    qou_features = []\n","    for f in features: \n","      qou_features.append(f / (len(cousers) ** 0.5))\n","      \n","        \n","    user_features = []\n","\n","    for f in ori_user_features:\n","      user_features.append(np.full(n, f))\n","\n","    item_features = get_item_features(coitems[order], all_item_feature_ids)\n","     \n","    level2_features = create_level2_data(np.transpose(np.stack((\n","                                            coitems[order],\n","                                            *item_features, \n","                                            *norm_features,\n","                                            *qou_features,\n","                                            *user_features,\n","                                            *cointeractions_sub_coef[:, order],\n","                                            *cointeractions_time_decay[:, order],\n","                                            ))))\n","  d = level2_features.merge(level1_features, on = 'item', how = 'left').fillna(0)\n","  assert d.shape[1] == len(level1_columns) + len(level2_columns) - 1, (d.shape[1], len(level1_columns), len(level2_columns))\n","  \n","  user_feats = pd.DataFrame(data = np.tile(np.ravel(user_features_df.loc[user].values).reshape(1,-1), (d.shape[0], 1)), columns = user_features_df.columns, copy = False)\n","  item_feats = pd.DataFrame({'aid': coitems[order] - MAX_SESSION}).set_index('aid')\n","  \n","  item_feats = item_feats.merge(item_features_df, left_index = True, right_index = True, how = 'left').fillna(0).rename(\n","    columns = dict(zip(item_features_df.columns, ['item_' + f for f in item_features_df.columns]))\n","  )\n","  \n","    \n","  d = pd.concat((d, user_feats, item_feats.reset_index(drop = True)), axis = 1)\n","  d.insert(loc = 0, column = 'user', value = user)\n","    \n","  assert len(np.unique(d.item)) == d.shape[0]\n","  \n","  \n","  data.append(d.astype('float32'))\n","    \n","    \n","\n","    \n"]},{"attachments":{},"cell_type":"markdown","metadata":{"id":"_9jtfSoRxFxU"},"source":["# Batchproc and Meow postprocess\n"]},{"cell_type":"code","execution_count":42,"metadata":{"id":"CBxvrd_axWcB"},"outputs":[],"source":["import traceback\n","from meow_utils import alpha_Meow_infer\n","import _pickle as pickle\n","def postprocess(rs, batch, test = False):\n","  global Meow, click_models, cart_models, order_models, GET_RAW\n","  infer_data, data_file, result_file = rs\n","  \n","  if GET_RAW:\n","    if not test:\n","      result = []\n","      for d in infer_data:\n","        # pred = d.sort_values('qou_lincom_time_decay_3_10_1_mean', ascending = False).item.tolist()[:200]\n","        result.append(d)\n","      assert len(result) == len(batch)\n","      result = [result for _ in range(3)]\n","      with open(result_file, 'wb') as f: \n","        pickle.dump(result, f)\n","      \n","      return\n","        \n","  infer_data = pd.concat(infer_data)\n","\n","        \n","  if Meow: \n","    result = []\n","    for models in [click_models, cart_models, order_models]:\n","      pred, user, probs = alpha_Meow_infer(infer_data, models, np.argsort(batch), top = 20 if JOIN else 200)\n","      assert np.array_equal(user, batch), (user[:10], batch[:10])\n","      result.append(pred)\n","      \n","    if not test:\n","      with open(result_file, 'wb') as f: \n","        pickle.dump(result, f)\n","\n","  if not test:\n","    \n","    infer_data.to_parquet(data_file, index = False) "]},{"cell_type":"code","execution_count":43,"metadata":{"id":"GKjs8q_6xIYD"},"outputs":[],"source":["import multiprocessing\n","import catboost\n","\n","def batch_proc(*args):\n","  global CACHE\n","  batch, batch_name = args[0]\n","\n","  data = []\n","  \n","  for user in batch:\n","    get_type_items(user, data)\n","\n","  postprocess([ data, CACHE + f'/data_{batch_name}.parquet', CACHE + f'/result_batch_{batch_name}'], batch)\n","  return batch\n","    "]},{"cell_type":"markdown","metadata":{"id":"vPHYwZUMRr5f"},"source":["# Infer Params"]},{"cell_type":"code","execution_count":44,"metadata":{"executionInfo":{"elapsed":12,"status":"ok","timestamp":1672118390113,"user":{"displayName":"Hải Minh Nguyễn","userId":"16371132940967469863"},"user_tz":-420},"id":"Uavnskh_RvtA"},"outputs":[],"source":["import multiprocessing\n","CACHE = '/home/anhphantq/vdb/cache_valid'\n","\n","MODEL_PATH = '/home/anhphantq/meowmeow'\n","MODEL_VERSION = '15'\n","\n","Meow = True\n","GET_RAW = False\n","\n","NUM_CPU = multiprocessing.cpu_count()\n","\n","# BATCH_SIZE = 4500\n","BATCH_SIZE = 140 * 5"]},{"cell_type":"code","execution_count":45,"metadata":{"executionInfo":{"elapsed":1368,"status":"ok","timestamp":1672118391473,"user":{"displayName":"Hải Minh Nguyễn","userId":"16371132940967469863"},"user_tz":-420},"id":"zjwx0XCU_-g1"},"outputs":[],"source":["!rm -rf {CACHE}\n","!mkdir {CACHE}"]},{"cell_type":"markdown","metadata":{"id":"qly6K4rcyg7Q"},"source":["# Create data batches"]},{"cell_type":"code","execution_count":46,"metadata":{"executionInfo":{"elapsed":289,"status":"ok","timestamp":1672118415035,"user":{"displayName":"Hải Minh Nguyễn","userId":"16371132940967469863"},"user_tz":-420},"id":"MlY2IYXNygVn","slideshow":{"slide_type":"slide"},"tags":[]},"outputs":[],"source":["from utils import shuffle, unshuffle"]},{"cell_type":"code","execution_count":47,"metadata":{"executionInfo":{"elapsed":5,"status":"ok","timestamp":1672118415544,"user":{"displayName":"Hải Minh Nguyễn","userId":"16371132940967469863"},"user_tz":-420},"id":"9c98rGgTynYO"},"outputs":[],"source":["import _pickle as pickle\n","perm = shuffle(session_list)\n","with open(CACHE + '/perm', 'wb') as f:\n","  pickle.dump(perm, f)"]},{"cell_type":"code","execution_count":48,"metadata":{"executionInfo":{"elapsed":5,"status":"ok","timestamp":1672118415545,"user":{"displayName":"Hải Minh Nguyễn","userId":"16371132940967469863"},"user_tz":-420},"id":"FQxwu4yiyqZj"},"outputs":[],"source":["import _pickle as pickle\n","\n","#divide into batches\n","\n","num_batches = len(session_list) // BATCH_SIZE\n","\n","start = np.arange(num_batches) * BATCH_SIZE\n","end = start + BATCH_SIZE\n","end[-1] = len(session_list)\n","batches = [session_list[start[i] : end[i]] for i in range(len(start))]\n","batch_names = [i for i in range(len(batches))]\n","\n","with open(CACHE + '/metadata', 'wb') as f:\n","  pickle.dump({\n","      'num_frags': len(batches), \n","      'max_session': MAX_SESSION,\n","  }, f)"]},{"attachments":{},"cell_type":"markdown","metadata":{"id":"gx5DT8ZmyyrR"},"source":["# Load Meow models"]},{"cell_type":"code","execution_count":52,"metadata":{"executionInfo":{"elapsed":523,"status":"ok","timestamp":1672119247563,"user":{"displayName":"Hải Minh Nguyễn","userId":"16371132940967469863"},"user_tz":-420},"id":"o9vgaJt3y1me"},"outputs":[],"source":["from catboost import CatBoostRanker\n","click_models = []\n","NUM_SPLITS = 5\n","NUM_SAMPLES = 3\n","NUM_CLICK_SAMPLES = 3\n","for fold in range(NUM_SPLITS):\n","  if fold == 2:\n","    break\n","  for i in range(NUM_CLICK_SAMPLES):\n","    click_model = CatBoostRanker()\n","    click_model.load_model(MODEL_PATH + '/clicks_only_' + MODEL_VERSION + f'/meowmewo_fold{fold}_sample{i}')\n","\n","    assert click_model.is_fitted\n","    click_models.append(click_model)\n","\n","cart_models = []\n","for fold in range(NUM_SPLITS):\n","  for i in range(NUM_SAMPLES):\n","    cart_model = CatBoostRanker()\n","    cart_model.load_model(MODEL_PATH + '/carts_only_'+ MODEL_VERSION + f'/meowmewo_fold{fold}_sample{i}')\n","    assert cart_model.is_fitted\n","    cart_models.append(cart_model)\n","\n","order_models = []\n","for fold in range(NUM_SPLITS):\n","  \n","  for i in range(NUM_SAMPLES):\n","    order_model = CatBoostRanker()\n","    order_model.load_model(MODEL_PATH + '/orders_only_'+ MODEL_VERSION + f'/meowmewo_fold{fold}_sample{i}')\n","    assert order_model.is_fitted\n","    order_models.append(order_model)\n","    \n","  "]},{"cell_type":"markdown","metadata":{"id":"bNPjlHMuM_1r"},"source":["# Test pipeline"]},{"cell_type":"code","execution_count":53,"metadata":{},"outputs":[],"source":["import pandas as pd\n","pd.set_option('display.max_columns', 484)"]},{"cell_type":"code","execution_count":54,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":851,"status":"ok","timestamp":1672108704106,"user":{"displayName":"Hải Minh Nguyễn","userId":"16371132940967469863"},"user_tz":-420},"id":"lM_77mtnNDx_","outputId":"2e5ebdc7-35ac-4335-f357-e5d8e46e6d00"},"outputs":[{"data":{"text/plain":["array([11152332], dtype=int32)"]},"execution_count":54,"metadata":{},"output_type":"execute_result"}],"source":["from tqdm.notebook import tqdm\n","# for user in tqdm(np.random.choice(session_list, size = 1000)):\n","batch = batch_proc((batches[0][10:11], [0]))\n","batch"]},{"cell_type":"markdown","metadata":{"id":"7cWxcscTOuub"},"source":["# Execute Multiprocessing"]},{"cell_type":"code","execution_count":55,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":0,"referenced_widgets":["197b31d9810644f48c3f60c6332aceaa","6bf9a456e2094eac863e1f0aad337187","fb252cb9db5a49dbaa9dc7e2d5d55a53","5889130120f24405a2102220358e9828","5cc6ae22ed064ab6b24fed13eacc5200","6d2358f791b649ababd4b23beef7d27c","50e4a9896b464d06957148414dd6fb66","2931156c14644222b8efb72b52e3602b","7272177e76c34d129237171d867c6eec","5657b4e116264782889b6763caaa4fe3","233119a38e1048208314f57ea2a3f346"]},"executionInfo":{"elapsed":10230,"status":"ok","timestamp":1672119273468,"user":{"displayName":"Hải Minh Nguyễn","userId":"16371132940967469863"},"user_tz":-420},"id":"tZ5vKPJuFuvt","outputId":"4f9c45c3-6cd4-4556-97e5-3eae000e3a6d"},"outputs":[{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"fd47d11e6e284f9d811ef3cdb9530b62","version_major":2,"version_minor":0},"text/plain":["CPU%:   0%|          | 0/100 [00:00<?, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"011835c83f644e7bb118a30d72d44bea","version_major":2,"version_minor":0},"text/plain":["RAM (GB):   0%|          | 0/125 [00:00<?, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"517c8e4301ce44cba884236a5af3301d","version_major":2,"version_minor":0},"text/plain":["Batches:   0%|          | 0/128 [00:00<?, ?it/s]"]},"metadata":{},"output_type":"display_data"},{"name":"stdout","output_type":"stream","text":["CPU times: user 782 ms, sys: 7.71 s, total: 8.49 s\n","Wall time: 6min 21s\n"]}],"source":["%%time\n","from tqdm.notebook import tqdm\n","import psutil\n","import _pickle as pickle\n","\n","with tqdm(total=100, desc='CPU%', position=1, leave = False) as cpubar, tqdm(total= int (psutil.virtual_memory().total  / 1024  / 1024 / 1024), desc='RAM (GB)', position=0, leave = False) as rambar:\n","                           \n","  with multiprocessing.Pool(NUM_CPU) as pool:\n","    rambar.n= int(psutil.virtual_memory().used / 1024  / 1024 / 1024)\n","\n","    cpubar.n=psutil.cpu_percent()\n","    rambar.refresh()\n","    cpubar.refresh()\n","    for  batch in tqdm(pool.imap_unordered(batch_proc, tuple(zip(batches, batch_names)), chunksize = 1), total = num_batches, desc = 'Batches'):\n","      # postprocess(rs, batch)\n","      \n","      rambar.n= int(psutil.virtual_memory().used / 1024  / 1024 / 1024)\n","\n","      cpubar.n=psutil.cpu_percent()\n","      rambar.refresh()\n","      cpubar.refresh()\n","\n"]},{"cell_type":"markdown","metadata":{"id":"tx0vY4Gn5fOB"},"source":["# Load result from pickle"]},{"cell_type":"code","execution_count":56,"metadata":{},"outputs":[],"source":["CACHE = '/home/anhphantq/vdb/cache_valid'\n"]},{"cell_type":"code","execution_count":57,"metadata":{"executionInfo":{"elapsed":13,"status":"ok","timestamp":1672119273980,"user":{"displayName":"Hải Minh Nguyễn","userId":"16371132940967469863"},"user_tz":-420},"id":"wzQUphH05izu"},"outputs":[],"source":["from utils import unshuffle"]},{"cell_type":"code","execution_count":58,"metadata":{"executionInfo":{"elapsed":13,"status":"ok","timestamp":1672119273981,"user":{"displayName":"Hải Minh Nguyễn","userId":"16371132940967469863"},"user_tz":-420},"id":"0rYvGvEcCAMa"},"outputs":[],"source":["import _pickle as pickle\n","with open(CACHE + '/metadata', 'rb') as f:\n","  metadata = pickle.load(f)\n","\n","num_frags = metadata['num_frags']\n","MAX_SESSION = metadata['max_session']"]},{"cell_type":"code","execution_count":59,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":0,"referenced_widgets":["0a21c74f10ac4b639a6f1ac314815e02","7e5f39795c82484599badb9a35840b85","539423a55091496aafd151610ee46018","b60170dedd34408a9ede05ee9734490f","e3b95b1d0e294e169d62841fe2d16d83","34c64419242e4e2691727ab51e2b21de","56e53fea7c034dc4ae50d3a90bee7ef1","bfe07d93ce2144909c8ee4a16fae5e6e","666147681b9442de972212bfe0dc213a","9aa5f28d1f1c48569a7e211add260b98","a5c0dd83f1a44fb886cf1ced5a0a321f"]},"executionInfo":{"elapsed":342,"status":"ok","timestamp":1672119274311,"user":{"displayName":"Hải Minh Nguyễn","userId":"16371132940967469863"},"user_tz":-420},"id":"NAOT802Ooiuu","outputId":"bedbf9c0-e795-4894-9e67-c5a3ef86d1af"},"outputs":[{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"555891e596a143e9bb214db0c158f518","version_major":2,"version_minor":0},"text/plain":["  0%|          | 0/128 [00:00<?, ?it/s]"]},"metadata":{},"output_type":"display_data"}],"source":["import multiprocessing\n","from tqdm.notebook import tqdm\n","\n","clicks = []\n","carts = []\n","orders = []\n","lens = []\n","for i in tqdm(range(num_frags)):\n","    with open(CACHE + f'/result_batch_{i}', 'rb') as f:\n","      result = pickle.load(f)\n","    \n","    clicks.extend(result[0])\n","    carts.extend(result[1])\n","    orders.extend(result[2])\n","    \n"]},{"cell_type":"code","execution_count":60,"metadata":{"executionInfo":{"elapsed":10,"status":"ok","timestamp":1672119274312,"user":{"displayName":"Hải Minh Nguyễn","userId":"16371132940967469863"},"user_tz":-420},"id":"_rieohF3BX5x"},"outputs":[],"source":["del result"]},{"cell_type":"code","execution_count":61,"metadata":{"executionInfo":{"elapsed":8,"status":"ok","timestamp":1672119274312,"user":{"displayName":"Hải Minh Nguyễn","userId":"16371132940967469863"},"user_tz":-420},"id":"yeWVPKIxu-NL"},"outputs":[],"source":["import _pickle as pickle\n","with open(CACHE + '/perm', 'rb') as f:\n","  perm = pickle.load(f)\n","unshuffle([clicks,carts,orders, session_list], perm)\n","# unshuffle([clicks,carts,orders], perm)"]},{"cell_type":"markdown","metadata":{"id":"-uZ6_jlXet75"},"source":["# Create submission"]},{"cell_type":"code","execution_count":62,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":491,"status":"ok","timestamp":1672119274796,"user":{"displayName":"Hải Minh Nguyễn","userId":"16371132940967469863"},"user_tz":-420},"id":"7Z3FvJ9Lmguz","outputId":"d3680d60-23b2-44ff-97ac-8606202923b8"},"outputs":[{"data":{"text/plain":["(270147, -15668151, 270186)"]},"execution_count":62,"metadata":{},"output_type":"execute_result"}],"source":["import pandas as pd\n","from utils import create_submission\n","count = 0\n","num_all = 0\n","missing = []\n","for preds in [clicks, carts, orders]:\n","  for i, p in enumerate(preds):\n","    if len(p) != 20:\n","      missing.append(i)\n","      count+= 20 - len(p)\n","    num_all +=1\n","\n","len(missing), count, num_all"]},{"cell_type":"code","execution_count":63,"metadata":{"executionInfo":{"elapsed":440,"status":"ok","timestamp":1672119275879,"user":{"displayName":"Hải Minh Nguyễn","userId":"16371132940967469863"},"user_tz":-420},"id":"285r_pbjIE74"},"outputs":[],"source":["final = create_submission(clicks, carts, orders, session_list, MAX_SESSION, JOIN)"]},{"cell_type":"code","execution_count":64,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":0},"executionInfo":{"elapsed":16,"status":"ok","timestamp":1672119275881,"user":{"displayName":"Hải Minh Nguyễn","userId":"16371132940967469863"},"user_tz":-420},"id":"5qAsoyDSOuaS","outputId":"c9d11cea-e0c9-4954-decf-235daf17b395"},"outputs":[{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>session_type</th>\n","      <th>labels</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>11098528_carts</td>\n","      <td>[11830, 1732105, 588923, 735729, 307904, 11578...</td>\n","    </tr>\n","    <tr>\n","      <th>0</th>\n","      <td>11098528_clicks</td>\n","      <td>[11830, 588923, 1732105, 1157882, 571762, 8761...</td>\n","    </tr>\n","    <tr>\n","      <th>0</th>\n","      <td>11098528_orders</td>\n","      <td>[11830, 1732105, 588923, 735729, 876129, 57176...</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>11098529_carts</td>\n","      <td>[1105029, 612829, 1049489, 333991, 1666485, 21...</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>11098529_clicks</td>\n","      <td>[1105029, 1715900, 459126, 217742, 333991, 159...</td>\n","    </tr>\n","    <tr>\n","      <th>...</th>\n","      <td>...</td>\n","      <td>...</td>\n","    </tr>\n","    <tr>\n","      <th>90060</th>\n","      <td>11188588_clicks</td>\n","      <td>[1347624, 1663535, 982423, 1582485, 784175, 49...</td>\n","    </tr>\n","    <tr>\n","      <th>90060</th>\n","      <td>11188588_orders</td>\n","      <td>[1663535, 982423, 1347624, 1582485, 493104, 57...</td>\n","    </tr>\n","    <tr>\n","      <th>90061</th>\n","      <td>11188589_carts</td>\n","      <td>[1627743, 641024, 912327, 1536424, 790171, 862...</td>\n","    </tr>\n","    <tr>\n","      <th>90061</th>\n","      <td>11188589_clicks</td>\n","      <td>[1627743, 641024, 195381, 862448, 924798, 9123...</td>\n","    </tr>\n","    <tr>\n","      <th>90061</th>\n","      <td>11188589_orders</td>\n","      <td>[1627743, 641024, 912327, 1536424, 790171, 195...</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>270186 rows × 2 columns</p>\n","</div>"],"text/plain":["          session_type                                             labels\n","0       11098528_carts  [11830, 1732105, 588923, 735729, 307904, 11578...\n","0      11098528_clicks  [11830, 588923, 1732105, 1157882, 571762, 8761...\n","0      11098528_orders  [11830, 1732105, 588923, 735729, 876129, 57176...\n","1       11098529_carts  [1105029, 612829, 1049489, 333991, 1666485, 21...\n","1      11098529_clicks  [1105029, 1715900, 459126, 217742, 333991, 159...\n","...                ...                                                ...\n","90060  11188588_clicks  [1347624, 1663535, 982423, 1582485, 784175, 49...\n","90060  11188588_orders  [1663535, 982423, 1347624, 1582485, 493104, 57...\n","90061   11188589_carts  [1627743, 641024, 912327, 1536424, 790171, 862...\n","90061  11188589_clicks  [1627743, 641024, 195381, 862448, 924798, 9123...\n","90061  11188589_orders  [1627743, 641024, 912327, 1536424, 790171, 195...\n","\n","[270186 rows x 2 columns]"]},"execution_count":64,"metadata":{},"output_type":"execute_result"}],"source":["final.sort_values('session_type')"]},{"cell_type":"markdown","metadata":{"id":"VaFgYG-LNeKR"},"source":["# Eval valid"]},{"cell_type":"code","execution_count":65,"metadata":{},"outputs":[],"source":["from utils import get_score, get_recall20"]},{"cell_type":"code","execution_count":66,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":4002,"status":"ok","timestamp":1672119279871,"user":{"displayName":"Hải Minh Nguyễn","userId":"16371132940967469863"},"user_tz":-420},"id":"ddU5rCEkOA_d","outputId":"068535ea-1db0-4709-90ed-eefea60ae60d"},"outputs":[{"name":"stdout","output_type":"stream","text":["Score : 0.5617045012293074\n"]},{"data":{"text/plain":["type\n","carts     0.407238\n","clicks    0.530682\n","orders    0.644108\n","dtype: float64"]},"execution_count":66,"metadata":{},"output_type":"execute_result"}],"source":["get_score(final, session_list, JOIN, n = 20)"]},{"cell_type":"code","execution_count":64,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["Index(['session', 'type', 'ground_truth'], dtype='object')\n"]},{"data":{"text/plain":["type\n","carts     0.546950\n","clicks    0.528711\n","orders    0.723792\n","Name: recall, dtype: float64"]},"execution_count":64,"metadata":{},"output_type":"execute_result"}],"source":["recall_per_type, test_labels = get_recall20(final, session_list, JOIN, n = 20)\n","recall_per_type"]},{"cell_type":"markdown","metadata":{"id":"1eBLOnamtWls"},"source":["# Save sumission"]},{"cell_type":"code","execution_count":60,"metadata":{"id":"tm7rgXdYIQXD"},"outputs":[],"source":["final.to_csv('submission.csv', index = False)"]},{"cell_type":"markdown","metadata":{"id":"ctpKcXOlInv4"},"source":["#Submit\n"]},{"cell_type":"code","execution_count":61,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":19490,"status":"ok","timestamp":1672087513875,"user":{"displayName":"Hải Minh Nguyễn","userId":"16371132940967469863"},"user_tz":-420},"id":"475yKZZPIt3D","outputId":"f41db76f-7a2e-46c0-b582-a3677d56d54b"},"outputs":[{"name":"stdout","output_type":"stream","text":["100%|████████████████████████████████████████| 782M/782M [00:47<00:00, 17.3MB/s]\n","Successfully submitted to OTTO – Multi-Objective Recommender System"]}],"source":["!kaggle competitions submit -c otto-recommender-system -f submission.csv -m \"hehe\""]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":[]}],"metadata":{"accelerator":"TPU","colab":{"authorship_tag":"ABX9TyNJ7f1pbb4MoIl018hXVUXn","collapsed_sections":["qcYhDDnhj_AJ","U7rFWhPZLfk5","o-jT6bO1pcOF","XNTi2k_UgUKa","lSWaAbtr_mlH","iUY4DIJiGsio","VelelmsEMYdG","tgE2wrB_54QH","IA7l7bDZlSTU","N3IpBw4N4jbO","036jHhe5U3F2","bNPjlHMuM_1r","tx0vY4Gn5fOB","-uZ6_jlXet75","VaFgYG-LNeKR","ctpKcXOlInv4","Y8HgtQVs0pnT","Vzl613PI-xzu","s5DAQhliT9UO"],"machine_shape":"hm","mount_file_id":"1uHVR7dW3R_x3Fq9NzIzROONyMnlQNX0R","provenance":[]},"gpuClass":"premium","kernelspec":{"display_name":"Python 3.8.10 64-bit","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.8.10 (default, Nov 14 2022, 12:59:47) \n[GCC 9.4.0]"},"vscode":{"interpreter":{"hash":"916dbcbb3f70747c44a77c7bcd40155683ae19c65e1c03b4aa3499c5328201f1"}},"widgets":{"application/vnd.jupyter.widget-state+json":{"0a21c74f10ac4b639a6f1ac314815e02":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_7e5f39795c82484599badb9a35840b85","IPY_MODEL_539423a55091496aafd151610ee46018","IPY_MODEL_b60170dedd34408a9ede05ee9734490f"],"layout":"IPY_MODEL_e3b95b1d0e294e169d62841fe2d16d83"}},"197b31d9810644f48c3f60c6332aceaa":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_6bf9a456e2094eac863e1f0aad337187","IPY_MODEL_fb252cb9db5a49dbaa9dc7e2d5d55a53","IPY_MODEL_5889130120f24405a2102220358e9828"],"layout":"IPY_MODEL_5cc6ae22ed064ab6b24fed13eacc5200"}},"233119a38e1048208314f57ea2a3f346":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"2931156c14644222b8efb72b52e3602b":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"34c64419242e4e2691727ab51e2b21de":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"50e4a9896b464d06957148414dd6fb66":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"539423a55091496aafd151610ee46018":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"FloatProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_bfe07d93ce2144909c8ee4a16fae5e6e","max":60,"min":0,"orientation":"horizontal","style":"IPY_MODEL_666147681b9442de972212bfe0dc213a","value":60}},"5657b4e116264782889b6763caaa4fe3":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"56e53fea7c034dc4ae50d3a90bee7ef1":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"5889130120f24405a2102220358e9828":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_5657b4e116264782889b6763caaa4fe3","placeholder":"​","style":"IPY_MODEL_233119a38e1048208314f57ea2a3f346","value":" 60/60 [00:03&lt;00:00, 25.39it/s]"}},"5cc6ae22ed064ab6b24fed13eacc5200":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"666147681b9442de972212bfe0dc213a":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"6bf9a456e2094eac863e1f0aad337187":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_6d2358f791b649ababd4b23beef7d27c","placeholder":"​","style":"IPY_MODEL_50e4a9896b464d06957148414dd6fb66","value":"100%"}},"6d2358f791b649ababd4b23beef7d27c":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"7272177e76c34d129237171d867c6eec":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"7e5f39795c82484599badb9a35840b85":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_34c64419242e4e2691727ab51e2b21de","placeholder":"​","style":"IPY_MODEL_56e53fea7c034dc4ae50d3a90bee7ef1","value":"100%"}},"9aa5f28d1f1c48569a7e211add260b98":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"a5c0dd83f1a44fb886cf1ced5a0a321f":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"b60170dedd34408a9ede05ee9734490f":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_9aa5f28d1f1c48569a7e211add260b98","placeholder":"​","style":"IPY_MODEL_a5c0dd83f1a44fb886cf1ced5a0a321f","value":" 60/60 [00:00&lt;00:00, 218.85it/s]"}},"bfe07d93ce2144909c8ee4a16fae5e6e":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"e3b95b1d0e294e169d62841fe2d16d83":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"fb252cb9db5a49dbaa9dc7e2d5d55a53":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"FloatProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_2931156c14644222b8efb72b52e3602b","max":60,"min":0,"orientation":"horizontal","style":"IPY_MODEL_7272177e76c34d129237171d867c6eec","value":60}}}}},"nbformat":4,"nbformat_minor":0}