ijcai-season2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import lightgbm as lgb\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import gc\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.metrics import log_loss\n",
    "from sklearn import preprocessing\n",
    "lbl = preprocessing.LabelEncoder()\n",
    "import warnings\n",
    "\n",
    "warnings.filterwarnings('ignore')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%run utils.py\n",
    "%run data_process.py"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "===========================single one===============================\n",
    "                   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_data_dir = '/home/hou/Windy/IJCAI18/fusai/data/sub0.txt'\n",
    "test_data_dir_1= '/home/hou/Windy/IJCAI18/data/round1_ijcai_18_test_a_20180301.txt'\n",
    "test_data_dir_2 = '/home/hou/Windy/IJCAI18/fusai/data/test.txt'\n",
    "test_save = pd.read_csv(test_data_dir_2,sep=\" \")\n",
    "test_save['model']=test_save['user_id'].apply(lambda x:x%10)\n",
    "test_save['predicted_score']=0\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_peng_feature(data):\n",
    "    print('peng_feature')\n",
    "    data['context_timestamp_str'] = data['context_timestamp'].astype(str)\n",
    "    user_time_join = data.groupby('user_id')['context_timestamp_str'].agg(lambda x:';'.join(x)).reset_index()\n",
    "    user_time_join.rename(columns={'context_timestamp_str':'user_time_join'},inplace = True)\n",
    "    data = pd.merge(data,user_time_join,on=['user_id'],how='left')\n",
    "    user_shop_time_join = data.groupby(['user_id','shop_id'])['context_timestamp_str'].agg(lambda x:';'.join(x)).reset_index()\n",
    "    user_shop_time_join.rename(columns={'context_timestamp_str':'user_shop_time_join'},inplace = True)\n",
    "    data = pd.merge(data,user_shop_time_join,on=['user_id','shop_id'],how='left')\n",
    "    user_item_time_join = data.groupby(['user_id','item_id'])['context_timestamp_str'].agg(lambda x:';'.join(x)).reset_index()\n",
    "    user_item_time_join.rename(columns={'context_timestamp_str':'user_item_time_join'},inplace = True)\n",
    "    data = pd.merge(data,user_item_time_join,on=['user_id','item_id'],how='left')\n",
    "    data['index_']=data.index\n",
    "    del user_time_join,user_shop_time_join,user_item_time_join\n",
    "\n",
    "    def get_20min_user_count(index_):\n",
    "        num=0\n",
    "        df1 = data.context_timestamp[index_]\n",
    "        df2 = data.user_time_join[index_].split(';')\n",
    "        df2_len = len(df2)\n",
    "        for i in range(df2_len):\n",
    "            if(((int(df2[i])-df1)<600)&((int(df2[i])-df1)>-600)):\n",
    "                num+=1\n",
    "        return num\n",
    "    def get_20min_user_shop_count(index_):\n",
    "        num=0\n",
    "        df1 = data.context_timestamp[index_]\n",
    "        df2 = data.user_shop_time_join[index_].split(';')\n",
    "        df2_len = len(df2)\n",
    "        for i in range(df2_len):\n",
    "            if(((int(df2[i])-df1)<600)&((int(df2[i])-df1)>-600)):\n",
    "                num+=1\n",
    "        return num\n",
    "    def get_20min_user_item_count(index_):\n",
    "        num=0\n",
    "        df1 = data.context_timestamp[index_]\n",
    "        df2 = data.user_item_time_join[index_].split(';')\n",
    "        df2_len = len(df2)\n",
    "        for i in range(df2_len):\n",
    "            if(((int(df2[i])-df1)<600)&((int(df2[i])-df1)>-600)):\n",
    "                num+=1\n",
    "        return num\n",
    "    a=time.time()\n",
    "    data['user_count_20']=data.index_.apply(get_20min_user_count)\n",
    "    print time.time()-a\n",
    "\n",
    "    a=time.time()\n",
    "    data['user_shop_count_20']=data.index_.apply(get_20min_user_shop_count)\n",
    "    print time.time()-a\n",
    "\n",
    "    a=time.time()\n",
    "    data['user_item_count_20']=data.index_.apply(get_20min_user_item_count)\n",
    "    print time.time()-a\n",
    "    return data\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#user item last one / all times\n",
    "#user category last one / all times\n",
    "def get_last_feature(data):\n",
    "    tt=time.time()\n",
    "    item_last = data[['user_id','item_id','day','context_timestamp_str']]\n",
    "    item_last = item_last.groupby(['day','user_id','item_id'])['context_timestamp_str'].agg(lambda x:';'.join(x)).reset_index()\n",
    "    item_last['user_item_day_num']=item_last.context_timestamp_str.apply(lambda s:len(s.split(';')))\n",
    "    data = pd.merge(data,item_last[['user_id','item_id','day','user_item_day_num']],on=['user_id','item_id','day'],how='left')\n",
    "   # print(data.shape)\n",
    "    item_last1 =item_last[item_last.user_item_day_num>1] \n",
    "   # print(item_last1.shape)\n",
    "    item_last1['max_time']=item_last1.context_timestamp_str.apply(lambda s:max([int(d) for d in s.split(';')]))\n",
    "   # print(item_last1.shape)\n",
    "    item_last1=item_last1[['user_id','item_id','day','max_time']]\n",
    "   # print(item_last1.shape)\n",
    "   # item_last = data.copy()\n",
    "   # print(data.shape)\n",
    "    data = pd.merge(data,item_last1,on=['user_id','item_id','day'],how='left')\n",
    "   # print(data.shape)\n",
    "    data['u_i_d_last_time']=data.max_time-data.context_timestamp_str.astype(int)\n",
    "   # print(data.shape)\n",
    "    def is_firstlastone(x):\n",
    "        if x==0:\n",
    "            return 1\n",
    "        elif x>0:\n",
    "            return 0\n",
    "        else :\n",
    "            return -1 #only receive once\n",
    "    data.u_i_d_last_time = data.u_i_d_last_time.apply(is_firstlastone)\n",
    "    #print(data.shape)\n",
    "    #data = pd.merge(data,item_last[['user_id','item_id','day','u_i_d_last_time']],on=['user_id','item_id','day'],how='left')\n",
    "    #print(data.shape)\n",
    "    cate_last = data[['user_id','cate_id','day','context_timestamp_str']]\n",
    "    cate_last = cate_last.groupby(['day','user_id','cate_id'])['context_timestamp_str'].agg(lambda x:';'.join(x)).reset_index()\n",
    "    cate_last['user_cate_day_num'] = cate_last.context_timestamp_str.apply(lambda s:len(s.split(';')))\n",
    "    data= pd.merge(data,cate_last[['user_id','cate_id','day','user_cate_day_num']],on=['user_id','cate_id','day'],how='left')\n",
    "    #print(data.shape)\n",
    "    cate_last1 = cate_last[cate_last.user_cate_day_num>1]\n",
    "    cate_last1['c_max_time']=cate_last1.context_timestamp_str.apply(lambda s:max([int(d) for d in s.split(';')]))\n",
    "    cate_last1=cate_last1[['user_id','cate_id','day','c_max_time']]\n",
    "    \n",
    "    #cate_last = data[['user_id','cate_id','day','context_timestamp_str']]\n",
    "    data = pd.merge(data,cate_last1,on=['user_id','cate_id','day'],how='left')\n",
    "    data['u_c_d_last_time'] = data.c_max_time-data.context_timestamp_str.astype(int)\n",
    "    data.u_c_d_last_time = data.u_c_d_last_time.apply(is_firstlastone)\n",
    "  #  data = pd.merge(data,cate_last[['user_id','cate_id','day','u_c_d_last_time']],on=['user_id','cate_id','day'],how='left')\n",
    "    print(data.shape)\n",
    "    print('get_last_time: ',time.time()-tt)\n",
    "\n",
    "    return data\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "start\n",
      "(1094123, 29)\n",
      "encode cate id now\n",
      "(1094123, 32)\n",
      "fill nan now\n",
      "(1094123, 32)\n",
      "encode now\n",
      "(1094123, 32)\n",
      "decode time\n",
      "(1094123, 34)\n",
      "cate_match\n",
      "(1094123, 40)\n",
      "tongji2\n",
      "(1094123, 53)\n",
      "peng_feature\n",
      "64.4940881729\n",
      "53.6697490215\n",
      "65.9731240273\n",
      "(1094123, 61)\n",
      "(1094123, 67)\n",
      "('get_last_time: ', 43.05160999298096)\n",
      "(1094123, 67)\n",
      "['cnt_shop_id', 'cnt_item_id', 'cnt_user_id', 'cnt_item_brand_id', 'cnt_item_city_id', 'cnt_user_id_item_id', 'cnt_user_id_shop_id', 'cnt_user_id_item_brand_id', 'cnt_user_id_item_city_id']\n",
      "431.570374966\n"
     ]
    }
   ],
   "source": [
    "print 'start'\n",
    "singletime=time.time()\n",
    "data=pd.read_csv(train_data_dir,sep=\" \")\n",
    "data=data.append(test_save[test_save.model==0]).reset_index().drop(['index'],axis=1)\n",
    "data['is_trade']=data['is_trade'].fillna(0)\n",
    "data = data.sort_values(by='context_timestamp').reset_index().drop(['index'],axis=1)\n",
    "gc.collect()\n",
    "print(data.shape)\n",
    "data=data_process0(data)\n",
    "print(data.shape)\n",
    "data=data_process1(data)\n",
    "print(data.shape)\n",
    "data=data_process2(data)\n",
    "print(data.shape)\n",
    "data=data_process3(data)\n",
    "print(data.shape)\n",
    "data=data_process4(data)\n",
    "print(data.shape)\n",
    "data=data_process6(data)#make some change from data_process5\n",
    "print(data.shape)\n",
    "data=get_peng_feature(data)\n",
    "print(data.shape)\n",
    "data=get_last_feature(data)\n",
    "print(data.shape)\n",
    "\n",
    "rank_list=['user_price_rank','user_sale_rank','user_collect_rank','pre_cate_price_rank','pre_cate_sale_rank','pre_cate_collect_rank']\n",
    "for col in rank_list:\n",
    "    if '%s_maxrank'%col in data.columns:\n",
    "        del data['%s_maxrank'%col]\n",
    "    tmp=data.groupby('user_day',as_index=False)[col].max().rename(columns={col:'%s_maxrank'%col})\n",
    "    data=data.merge(tmp,on='user_day',how='left')\n",
    "    data['%s_maxrank'%col]=data['%s_maxrank'%col].astype(float)\n",
    "    data[col]=data[col]/data['%s_maxrank'%col]\n",
    "    del data['%s_maxrank'%col]\n",
    "    \n",
    "day={4:data[data.day==4].copy(),5:data[data.day==5].copy(),\n",
    "    6:data[data.day==6].copy(),7:data[data.day==7].copy()}\n",
    "    \n",
    "count_list=[]\n",
    "for cols in [['shop_id'],['item_id'],['user_id'],['item_brand_id'],\n",
    "            ['item_city_id'],['user_id','item_id'],['user_id','shop_id'],['user_id','item_brand_id'],['user_id','item_city_id']]: \n",
    "    for day_v in xrange(4,8):#0,1,2,3is use for generate count\n",
    "        if len(cols)==1:\n",
    "            name='cnt_'+cols[0]\n",
    "        else:\n",
    "            name='cnt_'+cols[0]+'_'+cols[1]\n",
    "        day[day_v]=day[day_v].merge(get_silde_cnt(data,day_v,cols,name),on=cols,how='left')\n",
    "        day[day_v][name]=day[day_v][name].fillna(0)\n",
    "        gc.collect()\n",
    "    count_list+=[name]\n",
    "print count_list\n",
    "\n",
    "for day_v in xrange(4,8):\n",
    "    day[day_v]['item_user_ratio']=day[day_v]['cnt_user_id_item_id']/day[day_v]['cnt_user_id']\n",
    "    day[day_v]=get_count(day[day_v],['user_id'], 'cnt_user_tmp', 'is_trade')\n",
    "    day[day_v]=get_count(day[day_v],['user_id','item_id'], 'cnt_user_item_tmp', 'is_trade')\n",
    "    day[day_v]['item_user_ratio2']=day[day_v]['cnt_user_item_tmp']/day[day_v]['cnt_user_tmp']\n",
    "    del day[day_v]['cnt_user_tmp']\n",
    "gc.collect()\n",
    "print time.time()-singletime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1098983, 70)"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filter_day7=day[7].user_id%2==0\n",
    "train=pd.concat([day[4],day[5],day[6],day[7][(day[7].model.isnull())&(filter_day7)]]).copy().reset_index().drop('index',axis=1)\n",
    "val=day[7][(day[7].model.isnull())&(~filter_day7)].copy().reset_index().drop('index',axis=1)\n",
    "test=day[7][day[7].model.notnull()].copy().reset_index().drop('index',axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "canshu=1\n",
    "train['weight']=1\n",
    "train['weight'][train.day==7]=canshu\n",
    "train_weight=train['weight'].values\n",
    "train_weight_list = np.array(train_weight).tolist()\n",
    "\n",
    "val['weight']=1\n",
    "val['weight'][val.day==7]=canshu\n",
    "val_weight=val['weight'].values\n",
    "val_weight_list = np.array(val_weight).tolist()\n",
    "singletime=time.time()\n",
    "\n",
    "last_feature=['u_i_d_last_time','user_item_day_num','user_cate_day_num','u_c_d_last_time']\n",
    "peng_feature=['user_count_20','user_shop_count_20','user_item_count_20']\n",
    "cate_feature=['user_gender_id','user_occupation_id','cate_id','day']\n",
    "features=[ 'item_price_level', 'item_sales_level','item_collected_level','item_pv_level','user_age_level', 'user_star_level',\n",
    "         'shop_review_num_level', 'shop_star_level','shop_review_positive_rate', 'shop_score_service','shop_score_delivery', \n",
    "        'shop_score_description','cate_match','context_page_id', 'hour']+['item_user_ratio','item_user_ratio2','cnt_user_item_tmp',\n",
    "                  'pre_cate_price_rank','pre_cate_sale_rank','pre_cate_collect_rank','user_shop_cumcount',\n",
    "         'user_item_cumcount','shop_item_nunique','user_price_rank','user_sale_rank','user_collect_rank'                                                        \n",
    "         ]+\\\n",
    "        cate_feature+count_list\n",
    "    #+peng_feature+last_feature\n",
    "target='is_trade'\n",
    "\n",
    "\n",
    "\n",
    "gbm = lgb.LGBMClassifier(objective='binary',\n",
    "                                max_depth=5,\n",
    "                                num_leaves=16,\n",
    "                                learning_rate=0.02,\n",
    "                                n_estimators=2000,\n",
    "                                colsample_bytree=0.3,#0.3\n",
    "                                subsample = 0.75,#0.75\n",
    "                                n_jobs=4,\n",
    "                                lambda_l2=10,\n",
    "                                seed=2018,\n",
    "\n",
    "                                )\n",
    "gbm.fit(train[features],train[target],sample_weight=train_weight_list,categorical_feature=cate_feature,\n",
    "                    eval_set=[(val[features],val[target])],eval_sample_weight=None,\n",
    "                    eval_metric='binary_logloss',\n",
    "                    early_stopping_rounds=40,verbose=0)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "val_error=gbm.best_score_['valid_0']['binary_logloss']\n",
    "best_model=gbm.booster_\n",
    "train_pred=best_model.predict(train[features], num_iteration=best_model.best_iteration)\n",
    "print val_error,log_loss(train[target],train_pred)\n",
    "print time.time()-singletime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['item_price_level',\n",
       " 'item_sales_level',\n",
       " 'item_collected_level',\n",
       " 'item_pv_level',\n",
       " 'user_age_level',\n",
       " 'user_star_level',\n",
       " 'shop_review_num_level',\n",
       " 'shop_star_level',\n",
       " 'shop_review_positive_rate',\n",
       " 'shop_score_service',\n",
       " 'shop_score_delivery',\n",
       " 'shop_score_description',\n",
       " 'cate_match',\n",
       " 'context_page_id',\n",
       " 'hour',\n",
       " 'item_user_ratio',\n",
       " 'item_user_ratio2',\n",
       " 'cnt_user_item_tmp',\n",
       " 'pre_cate_price_rank',\n",
       " 'pre_cate_sale_rank',\n",
       " 'pre_cate_collect_rank',\n",
       " 'user_shop_cumcount',\n",
       " 'user_item_cumcount',\n",
       " 'shop_item_nunique',\n",
       " 'user_price_rank',\n",
       " 'user_sale_rank',\n",
       " 'user_collect_rank',\n",
       " 'user_gender_id',\n",
       " 'user_occupation_id',\n",
       " 'cate_id',\n",
       " 'day',\n",
       " 'cnt_shop_id',\n",
       " 'cnt_item_id',\n",
       " 'cnt_user_id',\n",
       " 'cnt_item_brand_id',\n",
       " 'cnt_item_city_id',\n",
       " 'cnt_user_id_item_id',\n",
       " 'cnt_user_id_shop_id',\n",
       " 'cnt_user_id_item_brand_id',\n",
       " 'cnt_user_id_item_city_id',\n",
       " 'user_count_20',\n",
       " 'user_shop_count_20',\n",
       " 'user_item_count_20']"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "=======================================all=================================================\n",
    "                                      "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_rate_feature(day7,data):\n",
    "    print(data.shape,day7.shape)\n",
    "    data=data[data.day!=7]\n",
    "    print(data.shape)\n",
    "    data2=data[['item_id','is_trade']]\n",
    "    data2['item_trade_all']=1\n",
    "    data2['item_trade_yes']=1\n",
    "    data2_all = data2.groupby(['item_id'],as_index=False).agg({'item_trade_all':'count'})\n",
    "    data2_yes = data2[data2.is_trade==1].groupby(['item_id'],as_index=False).agg({'item_trade_yes':'count'})\n",
    "    data2 = pd.merge(data2_all,data2_yes,on=['item_id'],how='left')\n",
    "    del data2_all,data2_yes\n",
    "    data2 = data2.fillna(0)\n",
    "    data2['item_rate']=(data2.item_trade_yes+0.001)/(data2.item_trade_all+0.001)\n",
    "    data2=data2[['item_id','item_rate','item_trade_all','item_trade_yes']]\n",
    "    day7 = pd.merge(day7,data2,on=['item_id'],how='left')\n",
    "    data3=data[['user_id','is_trade']]\n",
    "    data3['user_trade_all']=1\n",
    "    data3['user_trade_yes']=1\n",
    "    data3_all = data3.groupby(['user_id'],as_index=False).agg({'user_trade_all':'count'})\n",
    "    data3_yes = data3[data3.is_trade==1].groupby(['user_id'],as_index=False).agg({'user_trade_yes':'count'})\n",
    "    data3 = pd.merge(data3_all,data3_yes,on=['user_id'],how='left')\n",
    "    del data3_all,data3_yes\n",
    "    data3 = data3.fillna(0)\n",
    "    data3['user_rate']=(data3.user_trade_yes+0.001)/(data3.user_trade_all+0.001)\n",
    "    data3=data3[['user_id','user_rate','user_trade_all','user_trade_yes']]\n",
    "    day7 = pd.merge(day7,data3,on=['user_id'],how='left')\n",
    "    return day7\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_rate_feature_large(day7,data):\n",
    "    print(data.shape,day7.shape)\n",
    "    data=data[data.day!=7]\n",
    "    print(data.shape)\n",
    "    data2=data[['item_id','is_trade']]\n",
    "    data2['l_item_trade_all']=1\n",
    "    data2['l_item_trade_yes']=1\n",
    "    data2_all = data2.groupby(['item_id'],as_index=False).agg({'l_item_trade_all':'count'})\n",
    "    data2_yes = data2[data2.is_trade==1].groupby(['item_id'],as_index=False).agg({'l_item_trade_yes':'count'})\n",
    "    data2 = pd.merge(data2_all,data2_yes,on=['item_id'],how='left')\n",
    "    del data2_all,data2_yes\n",
    "    data2 = data2.fillna(0)\n",
    "    data2['l_item_rate']=(data2.l_item_trade_yes+0.001)/(data2.l_item_trade_all+0.001)\n",
    "    data2=data2[['item_id','l_item_rate','l_item_trade_all','l_item_trade_yes']]\n",
    "    day7 = pd.merge(day7,data2,on=['item_id'],how='left')\n",
    "    data3=data[['user_id','is_trade']]\n",
    "    data3['l_user_trade_all']=1\n",
    "    data3['l_user_trade_yes']=1\n",
    "    data3_all = data3.groupby(['user_id'],as_index=False).agg({'l_user_trade_all':'count'})\n",
    "    data3_yes = data3[data3.is_trade==1].groupby(['user_id'],as_index=False).agg({'l_user_trade_yes':'count'})\n",
    "    data3 = pd.merge(data3_all,data3_yes,on=['user_id'],how='left')\n",
    "    del data3_all,data3_yes\n",
    "    data3 = data3.fillna(0)\n",
    "    data3['l_user_rate']=(data3.l_user_trade_yes+0.001)/(data3.l_user_trade_all+0.001)\n",
    "    data3=data3[['user_id','l_user_rate','l_user_trade_all','l_user_trade_yes']]\n",
    "    day7 = pd.merge(day7,data3,on=['user_id'],how='left')\n",
    "    print day7.shape\n",
    "    return day7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "data 0\n",
      "(1094123, 29)\n",
      "encode cate id now\n",
      "(1094123, 32)\n",
      "fill nan now\n",
      "(1094123, 32)\n",
      "encode now\n",
      "(1094123, 32)\n",
      "decode time\n",
      "(1094123, 34)\n",
      "cate_match\n",
      "(1094123, 40)\n",
      "tongji2\n",
      "(1094123, 53)\n",
      "peng_feature\n",
      "64.4258580208\n",
      "49.5462400913\n",
      "48.9799849987\n",
      "(1094123, 61)\n",
      "(1094123, 67)\n",
      "('get_last_time: ', 40.768515825271606)\n",
      "['cnt_shop_id', 'cnt_item_id', 'cnt_user_id', 'cnt_item_brand_id', 'cnt_item_city_id', 'cnt_user_id_item_id', 'cnt_user_id_shop_id', 'cnt_user_id_item_brand_id', 'cnt_user_id_item_city_id']\n",
      "(159028, 79)\n",
      "((1094123, 67), (159028, 79))\n",
      "(935095, 67)\n",
      "(159028, 85)\n",
      "((159028, 85), (159028, 85))\n",
      "\n",
      "data 1\n",
      "(1097520, 29)\n",
      "encode cate id now\n",
      "(1097520, 32)\n",
      "fill nan now\n",
      "(1097520, 32)\n",
      "encode now\n",
      "(1097520, 32)\n",
      "decode time\n",
      "(1097520, 34)\n",
      "cate_match\n",
      "(1097520, 40)\n",
      "tongji2\n",
      "(1097520, 53)\n",
      "peng_feature\n",
      "64.242880106\n",
      "50.2037410736\n",
      "48.9366660118\n",
      "(1097520, 61)\n",
      "(1097520, 67)\n",
      "('get_last_time: ', 40.67762494087219)\n",
      "['cnt_shop_id', 'cnt_item_id', 'cnt_user_id', 'cnt_item_brand_id', 'cnt_item_city_id', 'cnt_user_id_item_id', 'cnt_user_id_shop_id', 'cnt_user_id_item_brand_id', 'cnt_user_id_item_city_id']\n",
      "(158676, 79)\n",
      "((1097520, 67), (158676, 79))\n",
      "(938844, 67)\n",
      "(158676, 85)\n",
      "((158676, 85), (317704, 85))\n",
      "\n",
      "data 2\n",
      "(1100005, 29)\n",
      "encode cate id now\n",
      "(1100005, 32)\n",
      "fill nan now\n",
      "(1100005, 32)\n",
      "encode now\n",
      "(1100005, 32)\n",
      "decode time\n",
      "(1100005, 34)\n",
      "cate_match\n",
      "(1100005, 40)\n",
      "tongji2\n",
      "(1100005, 53)\n",
      "peng_feature\n",
      "65.3626079559\n",
      "49.6363790035\n",
      "48.9342670441\n",
      "(1100005, 61)\n",
      "(1100005, 67)\n",
      "('get_last_time: ', 47.31074285507202)\n",
      "['cnt_shop_id', 'cnt_item_id', 'cnt_user_id', 'cnt_item_brand_id', 'cnt_item_city_id', 'cnt_user_id_item_id', 'cnt_user_id_shop_id', 'cnt_user_id_item_brand_id', 'cnt_user_id_item_city_id']\n",
      "(161247, 79)\n",
      "((1100005, 67), (161247, 79))\n",
      "(938758, 67)\n",
      "(161247, 85)\n",
      "((161247, 85), (478951, 85))\n",
      "\n",
      "data 3\n",
      "(1090416, 29)\n",
      "encode cate id now\n",
      "(1090416, 32)\n",
      "fill nan now\n",
      "(1090416, 32)\n",
      "encode now\n",
      "(1090416, 32)\n",
      "decode time\n",
      "(1090416, 34)\n",
      "cate_match\n",
      "(1090416, 40)\n",
      "tongji2\n",
      "(1090416, 53)\n",
      "peng_feature\n",
      "62.8847379684\n",
      "49.3178009987\n",
      "50.1626749039\n",
      "(1090416, 61)\n",
      "(1090416, 67)\n",
      "('get_last_time: ', 40.52680802345276)\n",
      "['cnt_shop_id', 'cnt_item_id', 'cnt_user_id', 'cnt_item_brand_id', 'cnt_item_city_id', 'cnt_user_id_item_id', 'cnt_user_id_shop_id', 'cnt_user_id_item_brand_id', 'cnt_user_id_item_city_id']\n",
      "(159185, 79)\n",
      "((1090416, 67), (159185, 79))\n",
      "(931231, 67)\n",
      "(159185, 85)\n",
      "((159185, 85), (638136, 85))\n",
      "\n",
      "data 4\n",
      "(1093944, 29)\n",
      "encode cate id now\n",
      "(1093944, 32)\n",
      "fill nan now\n",
      "(1093944, 32)\n",
      "encode now\n",
      "(1093944, 32)\n",
      "decode time\n",
      "(1093944, 34)\n",
      "cate_match\n",
      "(1093944, 40)\n",
      "tongji2\n",
      "(1093944, 53)\n",
      "peng_feature\n",
      "62.608273983\n",
      "48.8759980202\n",
      "48.2651619911\n",
      "(1093944, 61)\n",
      "(1093944, 67)\n",
      "('get_last_time: ', 40.13187599182129)\n",
      "['cnt_shop_id', 'cnt_item_id', 'cnt_user_id', 'cnt_item_brand_id', 'cnt_item_city_id', 'cnt_user_id_item_id', 'cnt_user_id_shop_id', 'cnt_user_id_item_brand_id', 'cnt_user_id_item_city_id']\n",
      "(160690, 79)\n",
      "((1093944, 67), (160690, 79))\n",
      "(933254, 67)\n",
      "(160690, 85)\n",
      "((160690, 85), (798826, 85))\n",
      "\n",
      "data 5\n",
      "(1098591, 29)\n",
      "encode cate id now\n",
      "(1098591, 32)\n",
      "fill nan now\n",
      "(1098591, 32)\n",
      "encode now\n",
      "(1098591, 32)\n",
      "decode time\n",
      "(1098591, 34)\n",
      "cate_match\n",
      "(1098591, 40)\n",
      "tongji2\n",
      "(1098591, 53)\n",
      "peng_feature\n",
      "62.5868160725\n",
      "48.5407011509\n",
      "48.3860878944\n",
      "(1098591, 61)\n",
      "(1098591, 67)\n",
      "('get_last_time: ', 40.73098611831665)\n",
      "['cnt_shop_id', 'cnt_item_id', 'cnt_user_id', 'cnt_item_brand_id', 'cnt_item_city_id', 'cnt_user_id_item_id', 'cnt_user_id_shop_id', 'cnt_user_id_item_brand_id', 'cnt_user_id_item_city_id']\n",
      "(159348, 79)\n",
      "((1098591, 67), (159348, 79))\n",
      "(939243, 67)\n",
      "(159348, 85)\n",
      "((159348, 85), (958174, 85))\n",
      "\n",
      "data 6\n",
      "(1091733, 29)\n",
      "encode cate id now\n",
      "(1091733, 32)\n",
      "fill nan now\n",
      "(1091733, 32)\n",
      "encode now\n",
      "(1091733, 32)\n",
      "decode time\n",
      "(1091733, 34)\n",
      "cate_match\n",
      "(1091733, 40)\n",
      "tongji2\n",
      "(1091733, 53)\n",
      "peng_feature\n",
      "61.6996309757\n",
      "48.4238710403\n",
      "48.0252158642\n",
      "(1091733, 61)\n",
      "(1091733, 67)\n",
      "('get_last_time: ', 40.42333984375)\n",
      "['cnt_shop_id', 'cnt_item_id', 'cnt_user_id', 'cnt_item_brand_id', 'cnt_item_city_id', 'cnt_user_id_item_id', 'cnt_user_id_shop_id', 'cnt_user_id_item_brand_id', 'cnt_user_id_item_city_id']\n",
      "(159446, 79)\n",
      "((1091733, 67), (159446, 79))\n",
      "(932287, 67)\n",
      "(159446, 85)\n",
      "((159446, 85), (1117620, 85))\n",
      "\n",
      "data 7\n",
      "(1097041, 29)\n",
      "encode cate id now\n",
      "(1097041, 32)\n",
      "fill nan now\n",
      "(1097041, 32)\n",
      "encode now\n",
      "(1097041, 32)\n",
      "decode time\n",
      "(1097041, 34)\n",
      "cate_match\n",
      "(1097041, 40)\n",
      "tongji2\n",
      "(1097041, 53)\n",
      "peng_feature\n",
      "63.4192428589\n",
      "49.4642350674\n",
      "49.4669549465\n",
      "(1097041, 61)\n",
      "(1097041, 67)\n",
      "('get_last_time: ', 41.66256380081177)\n",
      "['cnt_shop_id', 'cnt_item_id', 'cnt_user_id', 'cnt_item_brand_id', 'cnt_item_city_id', 'cnt_user_id_item_id', 'cnt_user_id_shop_id', 'cnt_user_id_item_brand_id', 'cnt_user_id_item_city_id']\n",
      "(160049, 79)\n",
      "((1097041, 67), (160049, 79))\n",
      "(936992, 67)\n",
      "(160049, 85)\n",
      "((160049, 85), (1277669, 85))\n",
      "\n",
      "data 8\n",
      "(1098983, 29)\n",
      "encode cate id now\n",
      "(1098983, 32)\n",
      "fill nan now\n",
      "(1098983, 32)\n",
      "encode now\n",
      "(1098983, 32)\n",
      "decode time\n",
      "(1098983, 34)\n",
      "cate_match\n",
      "(1098983, 40)\n",
      "tongji2\n",
      "(1098983, 53)\n",
      "peng_feature\n",
      "62.6476368904\n",
      "49.3338251114\n",
      "48.1274580956\n",
      "(1098983, 61)\n",
      "(1098983, 67)\n",
      "('get_last_time: ', 40.449613094329834)\n",
      "['cnt_shop_id', 'cnt_item_id', 'cnt_user_id', 'cnt_item_brand_id', 'cnt_item_city_id', 'cnt_user_id_item_id', 'cnt_user_id_shop_id', 'cnt_user_id_item_brand_id', 'cnt_user_id_item_city_id']\n",
      "(159542, 79)\n",
      "((1098983, 67), (159542, 79))\n",
      "(939441, 67)\n",
      "(159542, 85)\n",
      "((159542, 85), (1437211, 85))\n",
      "\n",
      "data 9\n",
      "(1089568, 29)\n",
      "encode cate id now\n",
      "(1089568, 32)\n",
      "fill nan now\n",
      "(1089568, 32)\n",
      "encode now\n",
      "(1089568, 32)\n",
      "decode time\n",
      "(1089568, 34)\n",
      "cate_match\n",
      "(1089568, 40)\n",
      "tongji2\n",
      "(1089568, 53)\n",
      "peng_feature\n",
      "62.566257\n",
      "48.88554883\n",
      "47.9825129509\n",
      "(1089568, 61)\n",
      "(1089568, 67)\n",
      "('get_last_time: ', 40.40609002113342)\n",
      "['cnt_shop_id', 'cnt_item_id', 'cnt_user_id', 'cnt_item_brand_id', 'cnt_item_city_id', 'cnt_user_id_item_id', 'cnt_user_id_shop_id', 'cnt_user_id_item_brand_id', 'cnt_user_id_item_city_id']\n",
      "(159852, 79)\n",
      "((1089568, 67), (159852, 79))\n",
      "(929716, 67)\n",
      "(159852, 85)\n",
      "((159852, 85), (1597063, 85))\n",
      "((1597063, 85), (0, 4))\n",
      "4041.48376298\n"
     ]
    }
   ],
   "source": [
    "timm=time.time()\n",
    "test_save=pd.read_csv('/home/hou/Windy/IJCAI18/fusai/data/test.txt',sep=\" \")\n",
    "test_save['model']=test_save['user_id'].apply(lambda x:x%10)\n",
    "test_save['predicted_score']=0\n",
    "\n",
    "for i in xrange(10):\n",
    "    print\n",
    "    print \"data\",i\n",
    "\n",
    "    data=pd.read_csv('/home/hou/Windy/IJCAI18/fusai/data/sub%d.txt'%i,sep=\" \")\n",
    "    data=data.append(test_save[test_save.model==i]).reset_index().drop(['index'],axis=1)\n",
    "    data['is_trade']=data['is_trade'].fillna(0)\n",
    "    data = data.sort_values(by='context_timestamp').reset_index().drop(['index'],axis=1)\n",
    "    gc.collect()\n",
    "    print(data.shape)\n",
    "    data=data_process0(data)\n",
    "    print(data.shape)\n",
    "    data=data_process1(data)\n",
    "    print(data.shape)\n",
    "    data=data_process2(data)\n",
    "    print(data.shape)\n",
    "    data=data_process3(data)\n",
    "    print(data.shape)\n",
    "    data=data_process4(data)\n",
    "    print(data.shape)\n",
    "    data=data_process6(data)#make some change from data_process5\n",
    "    print(data.shape)\n",
    "    data=get_peng_feature(data)\n",
    "    print(data.shape)\n",
    "    data=get_last_feature(data)\n",
    "\n",
    "    rank_list=['user_price_rank','user_sale_rank','user_collect_rank','pre_cate_price_rank','pre_cate_sale_rank','pre_cate_collect_rank']\n",
    "    for col in rank_list:\n",
    "        if '%s_maxrank'%col in data.columns:\n",
    "            del data['%s_maxrank'%col]\n",
    "        tmp=data.groupby('user_day',as_index=False)[col].max().rename(columns={col:'%s_maxrank'%col})\n",
    "        data=data.merge(tmp,on='user_day',how='left')\n",
    "        data['%s_maxrank'%col]=data['%s_maxrank'%col].astype(float)\n",
    "        data[col]=data[col]/data['%s_maxrank'%col]\n",
    "        del data['%s_maxrank'%col]\n",
    "\n",
    "        \n",
    "        \n",
    "\n",
    "    day={4:data[data.day==4].copy(),5:data[data.day==5].copy(),\n",
    "        6:data[data.day==6].copy(),7:data[data.day==7].copy()}\n",
    "\n",
    "    \n",
    "    \n",
    "    count_list=[]\n",
    "    for cols in [['shop_id'],['item_id'],['user_id'],['item_brand_id'],\n",
    "                ['item_city_id'],['user_id','item_id'],['user_id','shop_id'],['user_id','item_brand_id'],['user_id','item_city_id']]: \n",
    "        for day_v in xrange(4,8):#0,1,2,3is use for generate count\n",
    "            if len(cols)==1:\n",
    "                name='cnt_'+cols[0]\n",
    "            else:\n",
    "                name='cnt_'+cols[0]+'_'+cols[1]\n",
    "            day[day_v]=day[day_v].merge(get_silde_cnt(data,day_v,cols,name),on=cols,how='left')\n",
    "            day[day_v][name]=day[day_v][name].fillna(0)\n",
    "            gc.collect()\n",
    "        count_list+=[name]\n",
    "    print count_list\n",
    "\n",
    "    for day_v in xrange(4,8):\n",
    "        day[day_v]['item_user_ratio']=day[day_v]['cnt_user_id_item_id']/day[day_v]['cnt_user_id']\n",
    "        day[day_v]=get_count(day[day_v],['user_id'], 'cnt_user_tmp', 'is_trade')\n",
    "        day[day_v]=get_count(day[day_v],['user_id','item_id'], 'cnt_user_item_tmp', 'is_trade')\n",
    "        day[day_v]['item_user_ratio2']=day[day_v]['cnt_user_item_tmp']/day[day_v]['cnt_user_tmp']\n",
    "        del day[day_v]['cnt_user_tmp']\n",
    "    print(day[7].shape)\n",
    "    day[7]=get_rate_feature(day[7],data)\n",
    "    print(day[7].shape)\n",
    "    rate_save=data[['user_id','item_id','day','is_trade']]\n",
    "    if(i==0):\n",
    "        allday7=pd.DataFrame(columns=day[7].columns)\n",
    "    if(i==0):\n",
    "        rate_saveall=pd.DataFrame(columns=rate_save.columns)\n",
    "    allday7=pd.concat([allday7,day[7]])\n",
    "    rate_saveall=pd.concat([rate_saveall,rate_save])\n",
    "    print(day[7].shape,allday7.shape)\n",
    "    print(rate_saveall.shape,rate_save.shape)\n",
    "print(allday7.shape,rate_save.shape)\n",
    "allday7.to_csv('/home/hou/Windy/IJCAI18/fusai/data/day7rate.csv',index=False)\n",
    "rate_saveall.to_csv('/home/hou/Windy/IJCAI18/fusai/data/rate_saveall.csv',index=False)\n",
    "print time.time()-timm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1597063, 85)\n",
      "(519888, 85)\n",
      "(1077175, 85)\n",
      "(215435, 85) (861740, 85)\n"
     ]
    }
   ],
   "source": [
    "\n",
    "print allday7.shape\n",
    "allday7 = allday7.sample(frac=1.0)  # 全部打乱\n",
    "test=allday7[allday7.model.notnull()].copy().reset_index().drop('index',axis=1)\n",
    "print test.shape\n",
    "allday7=allday7[(allday7.model.isnull())].copy().reset_index().drop('index',axis=1) \n",
    "print allday7.shape\n",
    "cut_idx = int(round(0.2 * allday7.shape[0]))\n",
    "val, train = allday7.iloc[:cut_idx], allday7.iloc[cut_idx:]\n",
    "print val.shape,train.shape\n",
    "del allday7\n",
    "\n",
    " \n",
    "\n",
    "test.to_csv('/home/hou/Windy/IJCAI18/fusai/data/df_test.csv')\n",
    "train.to_csv('/home/hou/Windy/IJCAI18/fusai/data/df_train.csv')\n",
    "val.to_csv('/home/hou/Windy/IJCAI18/fusai/data/df_val.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "((10951924, 4), (861740, 85))\n",
      "(9354861, 4)\n",
      "(861740, 91)\n",
      "((10951924, 4), (519888, 85))\n",
      "(9354861, 4)\n",
      "(519888, 91)\n",
      "((10951924, 4), (215435, 85))\n",
      "(9354861, 4)\n",
      "(215435, 91)\n"
     ]
    }
   ],
   "source": [
    "# zhuan huan lv \n",
    "train=get_rate_feature_large(train,rate_saveall)\n",
    "test=get_rate_feature_large(test,rate_saveall)\n",
    "val=get_rate_feature_large(val,rate_saveall)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((519888, 80), (753840, 80), (323335, 80))"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_save=pd.read_csv('/home/hou/Windy/IJCAI18/fusai/data/test.txt',sep=\" \")\n",
    "test=pd.read_csv('/home/hou/Windy/IJCAI18/fusai/data/df_test.csv')\n",
    "train=pd.read_csv('/home/hou/Windy/IJCAI18/fusai/data/df_train.csv')\n",
    "val=pd.read_csv('/home/hou/Windy/IJCAI18/fusai/data/df_val.csv')\n",
    "test.shape,train.shape,val.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "308"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def geshicuowu(data):\n",
    "    for name in['item_price_level', 'item_sales_level', 'item_collected_level', 'item_pv_level', \n",
    "                'user_age_level', 'user_star_level', 'shop_review_num_level', 'shop_star_level', 'cate_match', \n",
    "                'context_page_id', 'hour', 'cnt_user_item_tmp', 'user_shop_cumcount', 'user_item_cumcount', \n",
    "                'shop_item_nunique', 'user_gender_id', 'user_occupation_id', 'cate_id', 'day', 'user_count_20',\n",
    "                'user_shop_count_20', 'user_item_count_20', 'u_i_d_last_time', 'user_item_day_num', 'user_cate_day_num',\n",
    "                'u_c_d_last_time']:\n",
    "        data[name]=data[name].astype(float)\n",
    "    print data.shape\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train = geshicuowu(train)\n",
    "test = geshicuowu(test)\n",
    "val=geshicuowu(val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "47\n",
      "Training until validation scores don't improve for 40 rounds.\n",
      "[20]\tvalid_0's binary_logloss: 0.457634\n",
      "[40]\tvalid_0's binary_logloss: 0.335256\n",
      "[60]\tvalid_0's binary_logloss: 0.266963\n",
      "[80]\tvalid_0's binary_logloss: 0.227763\n",
      "[100]\tvalid_0's binary_logloss: 0.205098\n",
      "[120]\tvalid_0's binary_logloss: 0.192169\n",
      "[140]\tvalid_0's binary_logloss: 0.184624\n",
      "[160]\tvalid_0's binary_logloss: 0.180176\n",
      "[180]\tvalid_0's binary_logloss: 0.177456\n",
      "[200]\tvalid_0's binary_logloss: 0.175797\n",
      "[220]\tvalid_0's binary_logloss: 0.17466\n",
      "[240]\tvalid_0's binary_logloss: 0.173884\n",
      "[260]\tvalid_0's binary_logloss: 0.173266\n",
      "[280]\tvalid_0's binary_logloss: 0.172785\n",
      "[300]\tvalid_0's binary_logloss: 0.172466\n",
      "[320]\tvalid_0's binary_logloss: 0.172166\n",
      "[340]\tvalid_0's binary_logloss: 0.171915\n",
      "[360]\tvalid_0's binary_logloss: 0.171718\n",
      "[380]\tvalid_0's binary_logloss: 0.171541\n",
      "[400]\tvalid_0's binary_logloss: 0.171397\n",
      "[420]\tvalid_0's binary_logloss: 0.17127\n",
      "[440]\tvalid_0's binary_logloss: 0.171151\n",
      "[460]\tvalid_0's binary_logloss: 0.171045\n",
      "[480]\tvalid_0's binary_logloss: 0.170959\n",
      "[500]\tvalid_0's binary_logloss: 0.170866\n",
      "[520]\tvalid_0's binary_logloss: 0.170779\n",
      "[540]\tvalid_0's binary_logloss: 0.170703\n",
      "[560]\tvalid_0's binary_logloss: 0.170635\n",
      "[580]\tvalid_0's binary_logloss: 0.170573\n",
      "[600]\tvalid_0's binary_logloss: 0.170515\n",
      "[620]\tvalid_0's binary_logloss: 0.17046\n",
      "[640]\tvalid_0's binary_logloss: 0.170415\n",
      "[660]\tvalid_0's binary_logloss: 0.170359\n",
      "[680]\tvalid_0's binary_logloss: 0.170322\n",
      "[700]\tvalid_0's binary_logloss: 0.170277\n",
      "[720]\tvalid_0's binary_logloss: 0.170235\n",
      "[740]\tvalid_0's binary_logloss: 0.170193\n",
      "[760]\tvalid_0's binary_logloss: 0.170157\n",
      "[780]\tvalid_0's binary_logloss: 0.170122\n",
      "[800]\tvalid_0's binary_logloss: 0.170087\n",
      "[820]\tvalid_0's binary_logloss: 0.170051\n",
      "[840]\tvalid_0's binary_logloss: 0.17002\n",
      "[860]\tvalid_0's binary_logloss: 0.169983\n",
      "[880]\tvalid_0's binary_logloss: 0.169957\n",
      "[900]\tvalid_0's binary_logloss: 0.169925\n",
      "[920]\tvalid_0's binary_logloss: 0.1699\n",
      "[940]\tvalid_0's binary_logloss: 0.169878\n",
      "[960]\tvalid_0's binary_logloss: 0.169853\n",
      "[980]\tvalid_0's binary_logloss: 0.169824\n",
      "[1000]\tvalid_0's binary_logloss: 0.1698\n",
      "[1020]\tvalid_0's binary_logloss: 0.169776\n",
      "[1040]\tvalid_0's binary_logloss: 0.169752\n",
      "[1060]\tvalid_0's binary_logloss: 0.169735\n",
      "[1080]\tvalid_0's binary_logloss: 0.169714\n",
      "[1100]\tvalid_0's binary_logloss: 0.169699\n",
      "[1120]\tvalid_0's binary_logloss: 0.169677\n",
      "[1140]\tvalid_0's binary_logloss: 0.169655\n",
      "[1160]\tvalid_0's binary_logloss: 0.169634\n",
      "[1180]\tvalid_0's binary_logloss: 0.169614\n",
      "[1200]\tvalid_0's binary_logloss: 0.169601\n",
      "[1220]\tvalid_0's binary_logloss: 0.16958\n",
      "[1240]\tvalid_0's binary_logloss: 0.169569\n",
      "[1260]\tvalid_0's binary_logloss: 0.169552\n",
      "[1280]\tvalid_0's binary_logloss: 0.169538\n",
      "[1300]\tvalid_0's binary_logloss: 0.169519\n",
      "[1320]\tvalid_0's binary_logloss: 0.169508\n",
      "[1340]\tvalid_0's binary_logloss: 0.169496\n",
      "[1360]\tvalid_0's binary_logloss: 0.169478\n",
      "[1380]\tvalid_0's binary_logloss: 0.169466\n",
      "[1400]\tvalid_0's binary_logloss: 0.169451\n",
      "[1420]\tvalid_0's binary_logloss: 0.169436\n",
      "[1440]\tvalid_0's binary_logloss: 0.169423\n",
      "[1460]\tvalid_0's binary_logloss: 0.169415\n",
      "[1480]\tvalid_0's binary_logloss: 0.169406\n",
      "[1500]\tvalid_0's binary_logloss: 0.169398\n",
      "[1520]\tvalid_0's binary_logloss: 0.169384\n",
      "[1540]\tvalid_0's binary_logloss: 0.169374\n",
      "[1560]\tvalid_0's binary_logloss: 0.169365\n",
      "[1580]\tvalid_0's binary_logloss: 0.169354\n",
      "[1600]\tvalid_0's binary_logloss: 0.169347\n",
      "[1620]\tvalid_0's binary_logloss: 0.169338\n",
      "[1640]\tvalid_0's binary_logloss: 0.169331\n",
      "[1660]\tvalid_0's binary_logloss: 0.169318\n",
      "[1680]\tvalid_0's binary_logloss: 0.169309\n",
      "[1700]\tvalid_0's binary_logloss: 0.169302\n",
      "[1720]\tvalid_0's binary_logloss: 0.169291\n",
      "[1740]\tvalid_0's binary_logloss: 0.169282\n",
      "[1760]\tvalid_0's binary_logloss: 0.169277\n",
      "[1780]\tvalid_0's binary_logloss: 0.169269\n",
      "[1800]\tvalid_0's binary_logloss: 0.169264\n",
      "[1820]\tvalid_0's binary_logloss: 0.169258\n",
      "[1840]\tvalid_0's binary_logloss: 0.169255\n",
      "[1860]\tvalid_0's binary_logloss: 0.16925\n",
      "[1880]\tvalid_0's binary_logloss: 0.169239\n",
      "[1900]\tvalid_0's binary_logloss: 0.16923\n",
      "[1920]\tvalid_0's binary_logloss: 0.169222\n",
      "[1940]\tvalid_0's binary_logloss: 0.169217\n",
      "[1960]\tvalid_0's binary_logloss: 0.169212\n",
      "[1980]\tvalid_0's binary_logloss: 0.169201\n",
      "[2000]\tvalid_0's binary_logloss: 0.169197\n",
      "[2020]\tvalid_0's binary_logloss: 0.169189\n",
      "[2040]\tvalid_0's binary_logloss: 0.169183\n",
      "[2060]\tvalid_0's binary_logloss: 0.16918\n",
      "[2080]\tvalid_0's binary_logloss: 0.169174\n",
      "[2100]\tvalid_0's binary_logloss: 0.169168\n",
      "[2120]\tvalid_0's binary_logloss: 0.169161\n",
      "[2140]\tvalid_0's binary_logloss: 0.169153\n",
      "[2160]\tvalid_0's binary_logloss: 0.169152\n",
      "[2180]\tvalid_0's binary_logloss: 0.169147\n",
      "[2200]\tvalid_0's binary_logloss: 0.169141\n",
      "[2220]\tvalid_0's binary_logloss: 0.169137\n",
      "[2240]\tvalid_0's binary_logloss: 0.169136\n",
      "[2260]\tvalid_0's binary_logloss: 0.169133\n",
      "[2280]\tvalid_0's binary_logloss: 0.169129\n",
      "[2300]\tvalid_0's binary_logloss: 0.169124\n",
      "[2320]\tvalid_0's binary_logloss: 0.169121\n",
      "[2340]\tvalid_0's binary_logloss: 0.169118\n",
      "[2360]\tvalid_0's binary_logloss: 0.169109\n",
      "[2380]\tvalid_0's binary_logloss: 0.169105\n",
      "[2400]\tvalid_0's binary_logloss: 0.169099\n",
      "[2420]\tvalid_0's binary_logloss: 0.169098\n",
      "[2440]\tvalid_0's binary_logloss: 0.169096\n",
      "[2460]\tvalid_0's binary_logloss: 0.16909\n",
      "[2480]\tvalid_0's binary_logloss: 0.169085\n",
      "[2500]\tvalid_0's binary_logloss: 0.169083\n",
      "[2520]\tvalid_0's binary_logloss: 0.169082\n",
      "[2540]\tvalid_0's binary_logloss: 0.16908\n",
      "[2560]\tvalid_0's binary_logloss: 0.169083\n",
      "[2580]\tvalid_0's binary_logloss: 0.169078\n",
      "[2600]\tvalid_0's binary_logloss: 0.169074\n",
      "[2620]\tvalid_0's binary_logloss: 0.169072\n",
      "[2640]\tvalid_0's binary_logloss: 0.16907\n",
      "[2660]\tvalid_0's binary_logloss: 0.169069\n",
      "[2680]\tvalid_0's binary_logloss: 0.169066\n",
      "[2700]\tvalid_0's binary_logloss: 0.169065\n",
      "[2720]\tvalid_0's binary_logloss: 0.169059\n",
      "[2740]\tvalid_0's binary_logloss: 0.169054\n",
      "[2760]\tvalid_0's binary_logloss: 0.169053\n",
      "[2780]\tvalid_0's binary_logloss: 0.169048\n",
      "[2800]\tvalid_0's binary_logloss: 0.169045\n",
      "[2820]\tvalid_0's binary_logloss: 0.169044\n",
      "[2840]\tvalid_0's binary_logloss: 0.169041\n",
      "[2860]\tvalid_0's binary_logloss: 0.169036\n",
      "[2880]\tvalid_0's binary_logloss: 0.169033\n",
      "[2900]\tvalid_0's binary_logloss: 0.169033\n",
      "[2920]\tvalid_0's binary_logloss: 0.169034\n",
      "Early stopping, best iteration is:\n",
      "[2886]\tvalid_0's binary_logloss: 0.169032\n",
      "0.1690315594332209\n",
      "0.16714165604501252\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "278"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "test_save=pd.read_csv('/home/hou/Windy/IJCAI18/fusai/data/test.txt',sep=\" \")\n",
    "test_save['model']=test_save['user_id'].apply(lambda x:x%10)\n",
    "test_save['predicted_score']=0\n",
    "\n",
    "gc.collect()\n",
    "rate_feature=['l_user_trade_all','l_user_trade_yes','l_item_trade_all','l_item_trade_yes']\n",
    "count_list=['cnt_shop_id', 'cnt_item_id', 'cnt_user_id', 'cnt_item_brand_id', 'cnt_item_city_id', 'cnt_user_id_item_id', 'cnt_user_id_shop_id', 'cnt_user_id_item_brand_id', 'cnt_user_id_item_city_id']\n",
    "last_feature=['u_i_d_last_time','user_item_day_num','user_cate_day_num','u_c_d_last_time']\n",
    "peng_feature=['user_count_20','user_shop_count_20','user_item_count_20']\n",
    "cate_feature=['user_gender_id','user_occupation_id','cate_id','day']\n",
    "features=[ 'item_price_level', 'item_sales_level','item_collected_level','item_pv_level','user_age_level', 'user_star_level',\n",
    "         'shop_review_num_level', 'shop_star_level','shop_review_positive_rate', 'shop_score_service','shop_score_delivery', \n",
    "        'shop_score_description','cate_match','context_page_id', 'hour']+['item_user_ratio','item_user_ratio2','cnt_user_item_tmp',\n",
    "                  'pre_cate_price_rank','pre_cate_sale_rank','pre_cate_collect_rank','user_shop_cumcount',\n",
    "         'user_item_cumcount','shop_item_nunique','user_price_rank','user_sale_rank','user_collect_rank'                                                        \n",
    "         ]+\\\n",
    "        cate_feature+count_list+peng_feature+last_feature\n",
    "target='is_trade'\n",
    "print len(features)\n",
    "\n",
    "\n",
    "gbm = lgb.LGBMClassifier(objective='binary',\n",
    "                            max_depth=5,\n",
    "                            num_leaves=16,\n",
    "                            learning_rate=0.02,\n",
    "                            n_estimators=20000,\n",
    "                            colsample_bytree=0.3,#0.3\n",
    "                            subsample = 0.75,#0.75\n",
    "                            n_jobs=4,\n",
    "                            lambda_l2=10,\n",
    "                            seed=2018\n",
    "                            )\n",
    "\n",
    "\n",
    "gbm.fit(train[features],train[target],sample_weight=None,categorical_feature=cate_feature,\n",
    "                eval_set=[(val[features],val[target])],eval_sample_weight=None,\n",
    "                eval_metric='binary_logloss',\n",
    "                early_stopping_rounds=40,verbose=20)\n",
    "\n",
    "val_error=gbm.best_score_['valid_0']['binary_logloss']\n",
    "print val_error\n",
    "best_model=gbm.booster_\n",
    "test['pred']=best_model.predict(test[features], num_iteration=best_model.best_iteration)\n",
    "test_save=test_save.merge(test[['instance_id','pred']],on='instance_id',how='left')\n",
    "# filter_t=test_save.model==i\n",
    "test_save.loc[:,'predicted_score']=test_save.loc[:,'pred'].values\n",
    "train_pred=best_model.predict(train[features], num_iteration=best_model.best_iteration)\n",
    "print log_loss(train[target],train_pred)\n",
    "gc.collect()\n",
    "#test_save[['instance_id','predicted_score']].to_csv('baseline_y456_1.txt', index=False, sep=' ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "test_save=pd.read_csv('/home/hou/Windy/IJCAI18/fusai/data/test.txt',sep=\" \")\n",
    "test_save['model']=test_save['user_id'].apply(lambda x:x%10)\n",
    "test_save['predicted_score']=0\n",
    "\n",
    "test['pred']=best_model.predict(test[features], num_iteration=best_model.best_iteration)\n",
    "test_save=test_save.merge(test[['instance_id','pred']],on='instance_id',how='left')\n",
    "# filter_t=test_save.model==i\n",
    "test_save.loc[:,'predicted_score']=test_save.loc[:,'pred'].values\n",
    "\n",
    "gc.collect()\n",
    "test_save[['instance_id','predicted_score']].to_csv('baseline_y456.txt', index=False, sep=' ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}