06_feature_merge.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "09620545-1d3a-4d1c-bc80-64acec01b7be",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "import warnings\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.model_selection import StratifiedKFold,GroupKFold\n",
    "from sklearn.decomposition import TruncatedSVD\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\n",
    "from tqdm import tqdm\n",
    "import gc\n",
    "import re\n",
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "import os\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\n",
    "from gensim.models import Word2Vec\n",
    "import polars as pl\n",
    "from pathlib import Path\n",
    "from glob import glob\n",
    "import json\n",
    "import joblib\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "9711d4c7-0f39-4f2d-bc33-551166804d88",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "with open('../IND-WhoIsWho/pid_to_info_all.json', 'r') as file:\n",
    "    pid = json.load(file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "dbc61404-3d10-41e9-ac13-16795eb67d2f",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "train = pd.read_feather('data/train.feather')\n",
    "valid = pd.read_feather('data/valid.feather')\n",
    "test = pd.read_feather('data/test.feather')\n",
    "\n",
    "train['dataset_mode'] = 0\n",
    "valid['dataset_mode'] = 1\n",
    "test['dataset_mode'] = 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "bf9e757c-8e7f-40e9-9455-ba18d83a8d3e",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.concat([train,valid,test]).reset_index(drop = True)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "069110bc-1003-4645-a83a-98bb4c7a2086",
   "metadata": {},
   "outputs": [],
   "source": [
    "temp = pd.read_feather('bge_m3_emb_title_feat.pkl')\n",
    "temp.columns = ['PID','autherID'] + [f'bge_m3_{f}' for f in temp.columns[2:]]\n",
    "data = data.merge(temp,on = ['PID','autherID'],how = 'left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "d00941e8-c9b0-49c3-b991-42eb096aa4df",
   "metadata": {},
   "outputs": [],
   "source": [
    "temp = pd.read_feather('feats/bge_m3_abstract_sim_feats.feather')\n",
    "temp.columns = ['PID','autherID'] + [f'bge_m3_{f}' for f in temp.columns[2:]]\n",
    "data = data.merge(temp,on = ['PID','autherID'],how = 'left')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "94547538-518d-4951-a15d-1d23ea68a4db",
   "metadata": {},
   "outputs": [],
   "source": [
    "temp = pd.read_feather('feats/abstract_tfidf_sim.feather')\n",
    "temp = temp.drop(['label','pid_abstract_sim','pid_title_sim', 'pid_title_sim_mean',\n",
    "       'pid_title_sim_max', 'pid_title_sim_min', 'pid_title_sim_std',\n",
    "       'pid_title_sim_median'],axis = 1)\n",
    "temp.columns = ['PID','autherID'] + [f'tfidf_{f}' for f in temp.columns[2:]]\n",
    "data = data.merge(temp,on = ['PID','autherID'],how = 'left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "ae1cd45c-98b7-4e35-bb26-bce42f306f40",
   "metadata": {},
   "outputs": [],
   "source": [
    "temp = pd.read_feather('feats/title_tfidf_sim.feather')\n",
    "temp = temp.drop(['label','pid_title_sim'],axis = 1)\n",
    "temp.columns = ['PID','autherID'] + [f'tfidf_{f}' for f in temp.columns[2:]]\n",
    "data = data.merge(temp,on = ['PID','autherID'],how = 'left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "adbc763a-38e4-44f5-87ee-b0ab26c392f5",
   "metadata": {},
   "outputs": [],
   "source": [
    "temp = pd.read_feather('feats/venue_tfidf_sim.feather')\n",
    "temp = temp.drop(['label','pid_venue_sim','pid_title_sim', 'pid_title_sim_mean',\n",
    "       'pid_title_sim_max', 'pid_title_sim_min', 'pid_title_sim_std',\n",
    "       'pid_title_sim_median', 'pid_abstract_sim', 'pid_abstract_sim_mean',\n",
    "       'pid_abstract_sim_max', 'pid_abstract_sim_min', 'pid_abstract_sim_std',\n",
    "       'pid_abstract_sim_median', 'pid_keyword_text_sim',\n",
    "       'pid_keyword_text_sim_mean', 'pid_keyword_text_sim_max',\n",
    "       'pid_keyword_text_sim_min', 'pid_keyword_text_sim_std',\n",
    "       'pid_keyword_text_sim_median',],axis = 1)\n",
    "temp.columns = ['PID','autherID'] + [f'tfidf_{f}' for f in temp.columns[2:]]\n",
    "data = data.merge(temp,on = ['PID','autherID'],how = 'left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "e1161f1d-9f4d-480a-9bba-3cece5b1ec9e",
   "metadata": {},
   "outputs": [],
   "source": [
    "temp = pd.read_feather('feats/keyword_text_tfidf_sim.feather')\n",
    "temp = temp.drop(['label','pid_keyword_text_sim', 'pid_title_sim', 'pid_title_sim_mean',\n",
    "       'pid_title_sim_max', 'pid_title_sim_min', 'pid_title_sim_std',\n",
    "       'pid_title_sim_median', 'pid_abstract_sim', 'pid_abstract_sim_mean',\n",
    "       'pid_abstract_sim_max', 'pid_abstract_sim_min', 'pid_abstract_sim_std',\n",
    "       'pid_abstract_sim_median'],axis = 1)\n",
    "temp.columns = ['PID','autherID'] + [f'tfidf_{f}' for f in temp.columns[2:]]\n",
    "data = data.merge(temp,on = ['PID','autherID'],how = 'left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "4137ba94-b474-48c7-bdcb-f6c454cdadad",
   "metadata": {},
   "outputs": [],
   "source": [
    "temp = pd.read_feather('feats/keywords_jaccard_sim.feather')\n",
    "temp = temp.drop(['label','pid_sim'],axis = 1)\n",
    "temp.columns = ['PID','autherID'] + [f'jaccard_{f}' for f in temp.columns[2:]]\n",
    "data = data.merge(temp,on = ['PID','autherID'],how = 'left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "dce01809-aefb-42c5-9d0a-d57fe96d32f2",
   "metadata": {},
   "outputs": [],
   "source": [
    "temp = pd.read_feather('feats/autherName_jaccard_sim.feather')\n",
    "temp = temp.drop(['label','pid_sim'],axis = 1)\n",
    "temp.columns = ['PID','autherID'] + [f'jaccard_{f}' for f in temp.columns[2:]]\n",
    "data = data.merge(temp,on = ['PID','autherID'],how = 'left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "3c5dbd8d-a710-4864-9d56-3f2190266b58",
   "metadata": {},
   "outputs": [],
   "source": [
    "temp = pd.read_feather('feats/autherName_raw_jaccard_sim.feather')\n",
    "temp = temp.drop(['label','pid_sim'],axis = 1)\n",
    "temp.columns = ['PID','autherID'] + [f'jaccard_{f}' for f in temp.columns[2:]]\n",
    "data = data.merge(temp,on = ['PID','autherID'],how = 'left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "48bf72d2-46d7-4e29-89bf-6cc3f4613c2b",
   "metadata": {},
   "outputs": [],
   "source": [
    "temp = pd.read_feather('feats/abstract_abstract2vec_feat.feather')\n",
    "data = data.merge(temp,on = ['PID'],how = 'left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "e33b9292-a9b2-4e4b-ac5e-d989d996f300",
   "metadata": {},
   "outputs": [],
   "source": [
    "temp = pd.read_feather('feats/title_tfidf2vec_feat.feather')\n",
    "data = data.merge(temp,on = ['PID'],how = 'left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "f88f7545-8497-4e13-badc-2b9d68a07824",
   "metadata": {},
   "outputs": [],
   "source": [
    "temp = pd.read_feather('feats/venue_venue2vec_feat.feather')\n",
    "data = data.merge(temp,on = ['PID'],how = 'left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "4699a836-3b2b-4af6-860c-6cb342b3b6ba",
   "metadata": {},
   "outputs": [],
   "source": [
    "temp = pd.read_feather('feats/keyword_text_keyword2vec_feat.feather')\n",
    "data = data.merge(temp,on = ['PID'],how = 'left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "2c95e8a6-21c3-4957-86a1-245a5b5c557f",
   "metadata": {},
   "outputs": [],
   "source": [
    "temp = pd.read_feather('feats/w2v_feats/autherID_keyword_w2v_emb.feather')\n",
    "data=  data.merge(temp,on = 'autherID',how = 'left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "96ed9791-ddc2-4cd6-85a9-012d9342b8a3",
   "metadata": {},
   "outputs": [],
   "source": [
    "temp = pd.read_feather('feats/w2v_feats/autherID_venue_w2v_emb.feather')\n",
    "data=  data.merge(temp,on = 'autherID',how = 'left')\n",
    "\n",
    "\n",
    "#temp = pd.read_feather('feats/w2v_feats/venue_autherID_w2v_emb_mean.feather')\n",
    "#data=  data.merge(temp,on = ['PID','autherID'],how = 'left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "7082aa97-f960-444e-a761-e2dfcb9da640",
   "metadata": {},
   "outputs": [],
   "source": [
    "#title,abstract\n",
    "t_dict = joblib.load('bge_m3_emb/abstract2vec_dict.pkl')\n",
    "abstract_emb = pd.DataFrame(t_dict.values()).values\n",
    "svd_tmp = TruncatedSVD(n_components=16, n_iter=20, random_state=2024)\n",
    "abstract_emb_svd = svd_tmp.fit_transform(abstract_emb)\n",
    "abstract_emb_svd = pd.DataFrame(abstract_emb_svd,columns = [f'abstract_emb_svd_{i}' for i in range(16)])\n",
    "abstract_emb_svd = pd.concat([pd.DataFrame(t_dict.keys(),columns = ['PID']),abstract_emb_svd],axis = 1)\n",
    "data = data.merge(abstract_emb_svd,on = ['PID'],how = 'left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "b6e885e5-2e15-4b48-90a2-8f3d0103b957",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "\n",
    "t_dict = joblib.load('bge_m3_emb/title2vec_dict.pkl')\n",
    "abstract_emb = pd.DataFrame(t_dict.values()).values\n",
    "svd_tmp = TruncatedSVD(n_components=16, n_iter=20, random_state=2024)\n",
    "abstract_emb_svd = svd_tmp.fit_transform(abstract_emb)\n",
    "abstract_emb_svd = pd.DataFrame(abstract_emb_svd,columns = [f'title_emb_svd_{i}' for i in range(16)])\n",
    "abstract_emb_svd = pd.concat([pd.DataFrame(t_dict.keys(),columns = ['PID']),abstract_emb_svd],axis = 1)\n",
    "data = data.merge(abstract_emb_svd,on = ['PID'],how = 'left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b7101713-70c5-4bed-83c4-28ef478a4e4b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "7e938072-3c6e-44dc-81fc-fdbf9ab92b51",
   "metadata": {},
   "outputs": [],
   "source": [
    "data['year'] = data['PID'].apply(lambda x:pid[x]['year'])\n",
    "data['year'] = data['year'].apply(lambda x:int(x) if type(x) == int else np.nan)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "bfd9e629-222b-4409-a068-7187950b3eb8",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "df = pd.read_feather('temp_data/autherID_authors.feather')\n",
    "\n",
    "df['index'] = df.index\n",
    "df['author_rank'] = df.groupby(['autherID','PID'])['index'].rank()\n",
    "\n",
    "temp = (df[df['flag'] == 1].groupby(['autherID','PID'])['author_rank'].mean()  / df.groupby(['autherID','PID'])['author_rank'].max()).reset_index()\n",
    "\n",
    "temp.columns = ['autherID','PID','author_rank_ratio']\n",
    "data = data.merge(temp,on = ['autherID','PID'],how = 'left')\n",
    "data = data.merge(df[df['flag'] == 1][['autherID','PID','author_rank']].groupby(['autherID','PID'])['author_rank'].mean().reset_index(),on = ['autherID','PID'],how = 'left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "e67deaac-83bc-4b13-9e2a-eba63c1311b7",
   "metadata": {},
   "outputs": [],
   "source": [
    "temp = pd.read_feather('feats/abstract_w2v_sim.feather')\n",
    "temp = temp.drop(['label','abstract_w2v_sim'],axis = 1)\n",
    "data = data.merge(temp ,on = ['autherID','PID'],how = 'left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "6f83e4ee-8b87-450a-a6ca-6d40939356b8",
   "metadata": {},
   "outputs": [],
   "source": [
    "temp = pd.read_feather('feats/title_w2v_sim.feather')\n",
    "temp = temp.drop(['label','title_w2v_sim'],axis = 1)\n",
    "data = data.merge(temp ,on = ['autherID','PID'],how = 'left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "c10a94c0-00da-4962-8ec2-01b7e67992d9",
   "metadata": {},
   "outputs": [],
   "source": [
    "temp = pd.read_feather('chatglm_pid_title_sim.pkl')\n",
    "data = data.merge(temp ,on = ['autherID','PID'],how = 'left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "c660e888-aaab-4b50-9abe-1aee5f631c4a",
   "metadata": {},
   "outputs": [],
   "source": [
    "temp = pd.read_feather('chatglm_pid_abstract_sim.pkl')\n",
    "data = data.merge(temp ,on = ['autherID','PID'],how = 'left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "cda8124a-83b3-4c93-91c4-7eebe4d2aa66",
   "metadata": {},
   "outputs": [],
   "source": [
    "temp = pd.read_feather('chatglm_pid_title_svd.pkl')\n",
    "data = data.merge(temp ,on = ['autherID','PID'],how = 'left')\n",
    "\n",
    "temp = pd.read_feather('chatglm_pid_abstract_svd.pkl')\n",
    "data = data.merge(temp ,on = ['autherID','PID'],how = 'left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "e8125821-2341-4b06-a0fc-b67902b05aa4",
   "metadata": {},
   "outputs": [],
   "source": [
    "data.to_feather('all_feat.feather')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "1247ce0a-627e-4cac-b33d-865402d28c17",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "372\n",
      "['bge_m3_pid_title_sim_mean', 'bge_m3_pid_title_sim_max', 'bge_m3_pid_title_sim_min', 'bge_m3_pid_title_sim_std', 'bge_m3_pid_title_sim_median', 'bge_m3_pid_abstract_sim_mean', 'bge_m3_pid_abstract_sim_max', 'bge_m3_pid_abstract_sim_min', 'bge_m3_pid_abstract_sim_std', 'bge_m3_pid_abstract_sim_median', 'tfidf_pid_abstract_sim_mean', 'tfidf_pid_abstract_sim_max', 'tfidf_pid_abstract_sim_min', 'tfidf_pid_abstract_sim_std', 'tfidf_pid_abstract_sim_median', 'tfidf_pid_title_sim_mean', 'tfidf_pid_title_sim_max', 'tfidf_pid_title_sim_min', 'tfidf_pid_title_sim_std', 'tfidf_pid_title_sim_median', 'tfidf_pid_venue_sim_mean', 'tfidf_pid_venue_sim_max', 'tfidf_pid_venue_sim_min', 'tfidf_pid_venue_sim_std', 'tfidf_pid_venue_sim_median', 'tfidf_pid_keyword_text_sim_mean', 'tfidf_pid_keyword_text_sim_max', 'tfidf_pid_keyword_text_sim_min', 'tfidf_pid_keyword_text_sim_std', 'tfidf_pid_keyword_text_sim_median', 'jaccard_pid_keyword_cnt_1_mean', 'jaccard_pid_keyword_cnt_1_max', 'jaccard_pid_keyword_cnt_1_min', 'jaccard_pid_keyword_cnt_1_std', 'jaccard_pid_keyword_cnt_1_median', 'jaccard_pid_keyword_cnt_2_mean', 'jaccard_pid_keyword_cnt_2_max', 'jaccard_pid_keyword_cnt_2_min', 'jaccard_pid_keyword_cnt_2_std', 'jaccard_pid_keyword_cnt_2_median', 'jaccard_pid_keyword_cnt_3_mean', 'jaccard_pid_keyword_cnt_3_max', 'jaccard_pid_keyword_cnt_3_min', 'jaccard_pid_keyword_cnt_3_std', 'jaccard_pid_keyword_cnt_3_median', 'jaccard_pid_auther_cnt_1_mean', 'jaccard_pid_auther_cnt_1_max', 'jaccard_pid_auther_cnt_1_min', 'jaccard_pid_auther_cnt_1_std', 'jaccard_pid_auther_cnt_2_mean', 'jaccard_pid_auther_cnt_2_max', 'jaccard_pid_auther_cnt_2_min', 'jaccard_pid_auther_cnt_2_std', 'jaccard_pid_auther_cnt_3_mean', 'jaccard_pid_auther_cnt_3_max', 'jaccard_pid_auther_cnt_3_min', 'jaccard_pid_auther_cnt_3_std', 'jaccard_pid_autherraw_cnt_1_mean', 'jaccard_pid_autherraw_cnt_1_max', 'jaccard_pid_autherraw_cnt_1_min', 'jaccard_pid_autherraw_cnt_1_std', 'jaccard_pid_autherraw_cnt_2_mean', 'jaccard_pid_autherraw_cnt_2_max', 'jaccard_pid_autherraw_cnt_2_min', 'jaccard_pid_autherraw_cnt_2_std', 'jaccard_pid_autherraw_cnt_3_mean', 'jaccard_pid_autherraw_cnt_3_max', 'jaccard_pid_autherraw_cnt_3_min', 'jaccard_pid_autherraw_cnt_3_std', 'abstract_tfidf_word_0', 'abstract_tfidf_word_1', 'abstract_tfidf_word_2', 'abstract_tfidf_word_3', 'abstract_tfidf_word_4', 'abstract_tfidf_word_5', 'abstract_tfidf_word_6', 'abstract_tfidf_word_7', 'abstract_tfidf_word_8', 'abstract_tfidf_word_9', 'abstract_tfidf_word_10', 'abstract_tfidf_word_11', 'abstract_tfidf_word_12', 'abstract_tfidf_word_13', 'abstract_tfidf_word_14', 'abstract_tfidf_word_15', 'abstract_tfidf_word_16', 'abstract_tfidf_word_17', 'abstract_tfidf_word_18', 'abstract_tfidf_word_19', 'abstract_tfidf_word_20', 'abstract_tfidf_word_21', 'abstract_tfidf_word_22', 'abstract_tfidf_word_23', 'abstract_tfidf_word_24', 'abstract_tfidf_word_25', 'abstract_tfidf_word_26', 'abstract_tfidf_word_27', 'abstract_tfidf_word_28', 'abstract_tfidf_word_29', 'abstract_tfidf_word_30', 'abstract_tfidf_word_31', 'abstract_countvec_0', 'abstract_countvec_1', 'abstract_countvec_2', 'abstract_countvec_3', 'abstract_countvec_4', 'abstract_countvec_5', 'abstract_countvec_6', 'abstract_countvec_7', 'abstract_countvec_8', 'abstract_countvec_9', 'abstract_countvec_10', 'abstract_countvec_11', 'abstract_countvec_12', 'abstract_countvec_13', 'abstract_countvec_14', 'abstract_countvec_15', 'abstract_countvec_16', 'abstract_countvec_17', 'abstract_countvec_18', 'abstract_countvec_19', 'abstract_countvec_20', 'abstract_countvec_21', 'abstract_countvec_22', 'abstract_countvec_23', 'abstract_countvec_24', 'abstract_countvec_25', 'abstract_countvec_26', 'abstract_countvec_27', 'abstract_countvec_28', 'abstract_countvec_29', 'abstract_countvec_30', 'abstract_countvec_31', 'title_tfidf_word_0', 'title_tfidf_word_1', 'title_tfidf_word_2', 'title_tfidf_word_3', 'title_tfidf_word_4', 'title_tfidf_word_5', 'title_tfidf_word_6', 'title_tfidf_word_7', 'title_tfidf_word_8', 'title_tfidf_word_9', 'title_tfidf_word_10', 'title_tfidf_word_11', 'title_tfidf_word_12', 'title_tfidf_word_13', 'title_tfidf_word_14', 'title_tfidf_word_15', 'title_tfidf_word_16', 'title_tfidf_word_17', 'title_tfidf_word_18', 'title_tfidf_word_19', 'title_tfidf_word_20', 'title_tfidf_word_21', 'title_tfidf_word_22', 'title_tfidf_word_23', 'title_tfidf_word_24', 'title_tfidf_word_25', 'title_tfidf_word_26', 'title_tfidf_word_27', 'title_tfidf_word_28', 'title_tfidf_word_29', 'title_tfidf_word_30', 'title_tfidf_word_31', 'title_countvec_0', 'title_countvec_1', 'title_countvec_2', 'title_countvec_3', 'title_countvec_4', 'title_countvec_5', 'title_countvec_6', 'title_countvec_7', 'title_countvec_8', 'title_countvec_9', 'title_countvec_10', 'title_countvec_11', 'title_countvec_12', 'title_countvec_13', 'title_countvec_14', 'title_countvec_15', 'title_countvec_16', 'title_countvec_17', 'title_countvec_18', 'title_countvec_19', 'title_countvec_20', 'title_countvec_21', 'title_countvec_22', 'title_countvec_23', 'title_countvec_24', 'title_countvec_25', 'title_countvec_26', 'title_countvec_27', 'title_countvec_28', 'title_countvec_29', 'title_countvec_30', 'title_countvec_31', 'venue_tfidf_word_0', 'venue_tfidf_word_1', 'venue_tfidf_word_2', 'venue_tfidf_word_3', 'venue_tfidf_word_4', 'venue_tfidf_word_5', 'venue_tfidf_word_6', 'venue_tfidf_word_7', 'venue_tfidf_word_8', 'venue_tfidf_word_9', 'venue_tfidf_word_10', 'venue_tfidf_word_11', 'venue_tfidf_word_12', 'venue_tfidf_word_13', 'venue_tfidf_word_14', 'venue_tfidf_word_15', 'venue_countvec_0', 'venue_countvec_1', 'venue_countvec_2', 'venue_countvec_3', 'venue_countvec_4', 'venue_countvec_5', 'venue_countvec_6', 'venue_countvec_7', 'venue_countvec_8', 'venue_countvec_9', 'venue_countvec_10', 'venue_countvec_11', 'venue_countvec_12', 'venue_countvec_13', 'venue_countvec_14', 'venue_countvec_15', 'keyword_text_tfidf_word_0', 'keyword_text_tfidf_word_1', 'keyword_text_tfidf_word_2', 'keyword_text_tfidf_word_3', 'keyword_text_tfidf_word_4', 'keyword_text_tfidf_word_5', 'keyword_text_tfidf_word_6', 'keyword_text_tfidf_word_7', 'keyword_text_tfidf_word_8', 'keyword_text_tfidf_word_9', 'keyword_text_tfidf_word_10', 'keyword_text_tfidf_word_11', 'keyword_text_tfidf_word_12', 'keyword_text_tfidf_word_13', 'keyword_text_tfidf_word_14', 'keyword_text_tfidf_word_15', 'keyword_text_countvec_0', 'keyword_text_countvec_1', 'keyword_text_countvec_2', 'keyword_text_countvec_3', 'keyword_text_countvec_4', 'keyword_text_countvec_5', 'keyword_text_countvec_6', 'keyword_text_countvec_7', 'keyword_text_countvec_8', 'keyword_text_countvec_9', 'keyword_text_countvec_10', 'keyword_text_countvec_11', 'keyword_text_countvec_12', 'keyword_text_countvec_13', 'keyword_text_countvec_14', 'keyword_text_countvec_15', 'autherID_keyword_emb_016', 'autherID_keyword_emb_116', 'autherID_keyword_emb_216', 'autherID_keyword_emb_316', 'autherID_keyword_emb_416', 'autherID_keyword_emb_516', 'autherID_keyword_emb_616', 'autherID_keyword_emb_716', 'autherID_keyword_emb_816', 'autherID_keyword_emb_916', 'autherID_keyword_emb_1016', 'autherID_keyword_emb_1116', 'autherID_keyword_emb_1216', 'autherID_keyword_emb_1316', 'autherID_keyword_emb_1416', 'autherID_keyword_emb_1516', 'autherID_venue_emb_08', 'autherID_venue_emb_18', 'autherID_venue_emb_28', 'autherID_venue_emb_38', 'autherID_venue_emb_48', 'autherID_venue_emb_58', 'autherID_venue_emb_68', 'autherID_venue_emb_78', 'abstract_emb_svd_0', 'abstract_emb_svd_1', 'abstract_emb_svd_2', 'abstract_emb_svd_3', 'abstract_emb_svd_4', 'abstract_emb_svd_5', 'abstract_emb_svd_6', 'abstract_emb_svd_7', 'abstract_emb_svd_8', 'abstract_emb_svd_9', 'abstract_emb_svd_10', 'abstract_emb_svd_11', 'abstract_emb_svd_12', 'abstract_emb_svd_13', 'abstract_emb_svd_14', 'abstract_emb_svd_15', 'title_emb_svd_0', 'title_emb_svd_1', 'title_emb_svd_2', 'title_emb_svd_3', 'title_emb_svd_4', 'title_emb_svd_5', 'title_emb_svd_6', 'title_emb_svd_7', 'title_emb_svd_8', 'title_emb_svd_9', 'title_emb_svd_10', 'title_emb_svd_11', 'title_emb_svd_12', 'title_emb_svd_13', 'title_emb_svd_14', 'title_emb_svd_15', 'year', 'author_rank_ratio', 'author_rank', 'abstract_w2v_sim_mean', 'abstract_w2v_sim_max', 'abstract_w2v_sim_min', 'abstract_w2v_sim_std', 'abstract_w2v_sim_median', 'title_w2v_sim_mean', 'title_w2v_sim_max', 'title_w2v_sim_min', 'title_w2v_sim_std', 'title_w2v_sim_median', 'chatglm_pid_title_sim_mean', 'chatglm_pid_title_sim_max', 'chatglm_pid_title_sim_min', 'chatglm_pid_title_sim_std', 'chatglm_pid_title_sim_median', 'chatglm_pid_abstract_sim_mean', 'chatglm_pid_abstract_sim_max', 'chatglm_pid_abstract_sim_min', 'chatglm_pid_abstract_sim_std', 'chatglm_pid_abstract_sim_median', 'chatglm_title_emb_svd_0', 'chatglm_title_emb_svd_1', 'chatglm_title_emb_svd_2', 'chatglm_title_emb_svd_3', 'chatglm_title_emb_svd_4', 'chatglm_title_emb_svd_5', 'chatglm_title_emb_svd_6', 'chatglm_title_emb_svd_7', 'chatglm_title_emb_svd_8', 'chatglm_title_emb_svd_9', 'chatglm_title_emb_svd_10', 'chatglm_title_emb_svd_11', 'chatglm_title_emb_svd_12', 'chatglm_title_emb_svd_13', 'chatglm_title_emb_svd_14', 'chatglm_title_emb_svd_15', 'chatglm_abstract_emb_svd_0', 'chatglm_abstract_emb_svd_1', 'chatglm_abstract_emb_svd_2', 'chatglm_abstract_emb_svd_3', 'chatglm_abstract_emb_svd_4', 'chatglm_abstract_emb_svd_5', 'chatglm_abstract_emb_svd_6', 'chatglm_abstract_emb_svd_7', 'chatglm_abstract_emb_svd_8', 'chatglm_abstract_emb_svd_9', 'chatglm_abstract_emb_svd_10', 'chatglm_abstract_emb_svd_11', 'chatglm_abstract_emb_svd_12', 'chatglm_abstract_emb_svd_13', 'chatglm_abstract_emb_svd_14', 'chatglm_abstract_emb_svd_15']\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "\n",
    "df_train = data[data['label'].notna()].reset_index(drop = True)\n",
    "df_test = data[data['label'].isna()].reset_index(drop = True)\n",
    "df_train['label'] = 1 - df_train['label']\n",
    "\n",
    "drop_feat =['tfidf_autherName','dataset_mode','id'] + ['PID','label','autherID','pid_title','autherName', 'abstract', 'keywords', 'venue', 'authors', 'index','cos_sim_pid','autherName_count'] \n",
    "used_feat = [f for f in df_train.columns if f not in (['PID','label','autherID'] + drop_feat )]\n",
    "print(len(used_feat))\n",
    "print(used_feat)\n",
    "ycol = 'label'\n",
    "\n",
    "#del data\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b40774aa-7073-4f12-b22d-381526264a8f",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}