whole_vs_trans.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "whole_file_A = 'result/ruc_A_whole_with_essay_directnoupdate_A1.txt'\n",
    "incre_file_A = 'result/ruc_A_incre_with_essay_direct_noupdate_A1.txt'\n",
    "whole_A = np.loadtxt(whole_file_A,delimiter=' ')\n",
    "incre_A = np.loadtxt(incre_file_A,delimiter=' ')\n",
    "whole_file_F = 'result/ruc_F_whole_with_essay_directnoupdate_A1.txt'\n",
    "incre_file_F = 'result/ruc_F_incre_with_essay_direct_noupdate_F1.txt'\n",
    "whole_F = np.loadtxt(whole_file_F,delimiter=' ')\n",
    "incre_F = np.loadtxt(incre_file_F,delimiter=' ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0.00159873  0.0038767   0.00784386 ...  0.0463837   0.\n",
      " -0.02307668]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([-2.13166644, -1.18985592, -0.7430355 , ..., 13.5116042 ,\n",
       "       17.39323071, 20.18846012])"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = whole_A[:,-5]\n",
    "print(a)\n",
    "np.sort(a)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "首先统计全量更新的a与增量更新的a的描述统计量。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DescribeResult(nobs=5359225, minmax=(-44.6488923337918, 48.73787331968845), mean=-0.0030447857389708267, variance=0.09609930001621907, skewness=-5.697969297950126, kurtosis=2389.6773746301715)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from scipy import stats\n",
    "stats.describe(whole_A,axis = None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DescribeResult(nobs=5080516, minmax=(-52.754441214793914, 42.028894135563164), mean=-0.004292982928312106, variance=0.060284908749264496, skewness=-10.099705362083899, kurtosis=3883.7904015416766)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stats.describe(incre_A, axis=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DescribeResult(nobs=317155, minmax=(-17.77617250142804, 20.188460122692987), mean=-0.015568071366383742, variance=0.08033313648131213, skewness=-10.333706631123222, kurtosis=995.0965894248354)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "incre_concept_num=137\n",
    "stats.describe(whole_A[:,-incre_concept_num:], axis=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DescribeResult(nobs=308798, minmax=(-31.143888685277084, 16.818443108410754), mean=-0.00460388911304811, variance=0.027954143742885372, skewness=-30.09427773749822, kurtosis=6955.687745631394)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "incre_course_num=137\n",
    "stats.describe(incre_A[:,-incre_course_num:], axis=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "operands could not be broadcast together with shapes (2254,) (2315,) ",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-13-344d0886160c>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msqrt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msquare\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mincre_A\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0mwhole_A\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m: operands could not be broadcast together with shapes (2254,) (2315,) "
     ]
    }
   ],
   "source": [
    "np.sqrt(np.sum(np.square(incre_A[:,-1]-whole_A[:,-1])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from preprocessing import read_file,row_normlize\n",
    "data_file = 'data/all_ruc_word_info_new_with_essay.csv'\n",
    "link_file = 'data/all_ruc_new_link_with_essay.csv'\n",
    "concept_file = 'data/ruc_all_concepts_new_with_essay.csv'\n",
    "file_type = 'sparse_row_col_val'\n",
    "X, links, concept = read_file(data_file, link_file, concept_file, file_type)\n",
    "X = X.todense()\n",
    "X = row_normlize(X)\n",
    "# incre_F1 = np.matmul(np.matmul(X[:,:-14],incre_A1),X[:,:-14].transpose())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2240, 329)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X[:,:-14].transpose().shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_course = X.shape[0]\n",
    "undirect = False\n",
    "from preprocessing import generate_triple, generate_trn\n",
    "trn = generate_trn(links, n_course, undirect=undirect)\n",
    "tripple = generate_triple(trn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('pre concept of {}:'.format(concept[-1]))\n",
    "print('left is whole-version, right is incremental-version\\n')\n",
    "for i,j in zip(s1,s2):\n",
    "    print(\"{}:{:.4f}\\t\\t{}:{:.4f}\".format(concept[i],whole_A[:,-1][i],concept[j],incre_A[:,-1][j]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "训练集auc(liu)为0.9942288487305977\n",
      "训练集auc为0.9701609090090979\n",
      "训练集auc(liu)为0.9287438461963095\n",
      "训练集auc为0.8851267543382045\n"
     ]
    },
    {
     "ename": "NameError",
     "evalue": "name 'incre_F1' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-10-03657bb8c47e>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'训练集auc为{0}'\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel_eval\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mauc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrn\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mevaluation\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m \u001b[0mmodel_eval\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mevaluation\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mincre_F1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mtripple\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     11\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'训练集auc(liu)为{0}'\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel_eval\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mauc_liu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'训练集auc为{0}'\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel_eval\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mauc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrn\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'incre_F1' is not defined"
     ]
    }
   ],
   "source": [
    "from evaluation import *\n",
    "model_eval = evaluation(whole_F,tripple)\n",
    "print('训练集auc(liu)为{0}'.format(model_eval.auc_liu()))\n",
    "print('训练集auc为{0}'.format(model_eval.auc(trn)))\n",
    "from evaluation import *\n",
    "model_eval = evaluation(incre_F,tripple)\n",
    "print('训练集auc(liu)为{0}'.format(model_eval.auc_liu()))\n",
    "print('训练集auc为{0}'.format(model_eval.auc(trn)))\n",
    "from evaluation import *\n",
    "model_eval = evaluation(incre_F1,tripple)\n",
    "print('训练集auc(liu)为{0}'.format(model_eval.auc_liu()))\n",
    "print('训练集auc为{0}'.format(model_eval.auc(trn)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = np.loadtxt('result/ruc_A_cgl.txt',delimiter=' ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "a[1252]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"{:.2f}\".format(3.1415926))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_course_pairs(F,percentile = 99.5):\n",
    "    threshold = np.percentile(F,percentile)\n",
    "    edge = np.argwhere(F>threshold)\n",
    "    edge = edge[edge[:,0] != edge[:,1]]\n",
    "    node = []\n",
    "    for i in range(len(edge)):\n",
    "        for j in range(2):\n",
    "            node.append(edge[i,j])\n",
    "    weight = np.round(F[edge[:,0],edge[:,1]],decimals = 2)\n",
    "    weight.shape = (weight.shape[0],1)\n",
    "    weight_edge = np.append(edge[:,[1,0]],weight,axis = 1)\n",
    "    weight_edge_list = [i.tolist() for i in weight_edge]\n",
    "    return(weight_edge_list)\n",
    "weight_edge_list = generate_course_pairs(incre_F, percentile=90)\n",
    "weight_edge_list = [i for i in weight_edge_list if i[0] in [151,152,153] or i[1] in [151,152,153]]\n",
    "name_edge_list = []\n",
    "chp_ids = []\n",
    "with open('data/ruc_book_chapter_ids.csv','r') as f:\n",
    "    for i in f:\n",
    "        chp_name = i.strip().split(',')[0]\n",
    "        chp_ids.append(chp_name)\n",
    "for i in weight_edge_list:\n",
    "    name_edge_list.append([chp_ids[int(i[0])],chp_ids[int(i[1])],round(i[2],3)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "86\n",
      "86\n",
      "[['12ConfidenceRegionLowRankRegre', '1StatisticalModels', 0.49], ['13PRCA', '1StatisticalModels', 0.54], ['12ConfidenceRegionLowRankRegre', '2StatisticalModels', 0.57], ['13PRCA', '2StatisticalModels', 0.58], ['12ConfidenceRegionLowRankRegre', '3StatisticalModels', 0.68], ['13PRCA', '3StatisticalModels', 0.66], ['13PRCA', '6StatisticalModels', 0.54], ['12ConfidenceRegionLowRankRegre', '8StatisticalModels', 0.55], ['13PRCA', '8StatisticalModels', 0.59], ['12ConfidenceRegionLowRankRegre', '9StatisticalModels', 0.56], ['13PRCA', '9StatisticalModels', 0.52], ['12ConfidenceRegionLowRankRegre', '1ConvexOptimizationbook', 0.52], ['13PRCA', '1ConvexOptimizationbook', 1.06], ['12ConfidenceRegionLowRankRegre', '2ConvexOptimizationbook', 0.49], ['13PRCA', '2ConvexOptimizationbook', 0.82], ['12ConfidenceRegionLowRankRegre', '3ConvexOptimizationbook', 0.53], ['13PRCA', '3ConvexOptimizationbook', 0.84], ['12ConfidenceRegionLowRankRegre', '4ConvexOptimizationbook', 0.74], ['13PRCA', '4ConvexOptimizationbook', 1.11], ['12ConfidenceRegionLowRankRegre', '5ConvexOptimizationbook', 0.78], ['13PRCA', '5ConvexOptimizationbook', 1.09], ['13PRCA', '6ConvexOptimizationbook', 0.53], ['12ConfidenceRegionLowRankRegre', '7ConvexOptimizationbook', 0.65], ['13PRCA', '7ConvexOptimizationbook', 0.8], ['12ConfidenceRegionLowRankRegre', '8ConvexOptimizationbook', 0.95], ['13PRCA', '8ConvexOptimizationbook', 1.14], ['12ConfidenceRegionLowRankRegre', '9ConvexOptimizationbook', 0.82], ['13PRCA', '9ConvexOptimizationbook', 0.99], ['12ConfidenceRegionLowRankRegre', '10ConvexOptimizationbook', 0.78], ['13PRCA', '10ConvexOptimizationbook', 1.12], ['12ConfidenceRegionLowRankRegre', '11ConvexOptimizationbook', 0.51], ['13PRCA', '11ConvexOptimizationbook', 0.8], ['12ConfidenceRegionLowRankRegre', '1RegressionModelingStrategies', 0.79], ['13PRCA', '1RegressionModelingStrategies', 0.75], ['12ConfidenceRegionLowRankRegre', '2RegressionModelingStrategies', 0.54], ['12ConfidenceRegionLowRankRegre', '3RegressionModelingStrategies', 0.82], ['13PRCA', '3RegressionModelingStrategies', 0.89], ['12ConfidenceRegionLowRankRegre', '4RegressionModelingStrategies', 0.82], ['13PRCA', '4RegressionModelingStrategies', 0.78], ['12ConfidenceRegionLowRankRegre', '5RegressionModelingStrategies', 0.55], ['13PRCA', '5RegressionModelingStrategies', 0.48], ['12ConfidenceRegionLowRankRegre', '8RegressionModelingStrategies', 0.84], ['13PRCA', '8RegressionModelingStrategies', 0.81], ['12ConfidenceRegionLowRankRegre', '11RegressionModelingStrategies', 0.63], ['13PRCA', '11RegressionModelingStrategies', 0.61], ['12ConfidenceRegionLowRankRegre', '15RegressionModelingStrategies', 0.56], ['12ConfidenceRegionLowRankRegre', '16RegressionModelingStrategies', 0.66], ['13PRCA', '2ComputationalStatistics', 0.5], ['12ConfidenceRegionLowRankRegre', '12ComputationalStatistics', 0.49], ['12ConfidenceRegionLowRankRegre', '1RossProbability', 0.53], ['12ConfidenceRegionLowRankRegre', '2RossProbability', 1.22], ['13PRCA', '2RossProbability', 1.1], ['12ConfidenceRegionLowRankRegre', '3RossProbability', 1.13], ['13PRCA', '3RossProbability', 1.58], ['12ConfidenceRegionLowRankRegre', '4RossProbability', 0.9], ['13PRCA', '4RossProbability', 0.97], ['12ConfidenceRegionLowRankRegre', '5RossProbability', 0.7], ['13PRCA', '5RossProbability', 0.69], ['12ConfidenceRegionLowRankRegre', '6RossProbability', 0.53], ['13PRCA', '6RossProbability', 0.82], ['12ConfidenceRegionLowRankRegre', '7RossProbability', 0.76], ['13PRCA', '7RossProbability', 0.95], ['12ConfidenceRegionLowRankRegre', '8RossProbability', 0.8], ['13PRCA', '8RossProbability', 0.91], ['12ConfidenceRegionLowRankRegre', '9RossProbability', 0.85], ['13PRCA', '9RossProbability', 0.89], ['12ConfidenceRegionLowRankRegre', '10RossProbability', 0.61], ['13PRCA', '10RossProbability', 0.64], ['13PRCA', '2CategoricalDataAnalysis', 0.58], ['13PRCA', '2DeepLearning', 0.94], ['12ConfidenceRegionLowRankRegre', '3DeepLearning', 0.58], ['13PRCA', '3DeepLearning', 0.75], ['13PRCA', '4DeepLearning', 0.7], ['13PRCA', '8DeepLearning', 0.58], ['13PRCA', '13DeepLearning', 0.5], ['13PRCA', '16DeepLearning', 0.61], ['13PRCA', '11L2RM', 0.53], ['8StatisticalModels', '12ConfidenceRegionLowRankRegre', 0.52], ['9StatisticalModels', '12ConfidenceRegionLowRankRegre', 0.62], ['8RegressionModelingStrategies', '12ConfidenceRegionLowRankRegre', 0.81], ['13PRCA', '12ConfidenceRegionLowRankRegre', 0.59], ['8ConvexOptimizationbook', '13PRCA', 0.54], ['8RegressionModelingStrategies', '13PRCA', 0.62], ['12RegressionModelingStrategies', '13PRCA', 0.53], ['4DeepLearning', '13PRCA', 0.6], ['11DeepLearning', '13PRCA', 0.5]]\n"
     ]
    }
   ],
   "source": [
    "print(len(weight_edge_list))\n",
    "print(len(name_edge_list))\n",
    "print(name_edge_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(153, 153)"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "whole_F.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "117.045"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "153*153*0.005"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['1StatisticalModels',\n",
       " '2StatisticalModels',\n",
       " '3StatisticalModels',\n",
       " '4StatisticalModels',\n",
       " '5StatisticalModels',\n",
       " '6StatisticalModels',\n",
       " '7StatisticalModels',\n",
       " '8StatisticalModels',\n",
       " '9StatisticalModels',\n",
       " '10StatisticalModels',\n",
       " '11StatisticalModels',\n",
       " '12StatisticalModels',\n",
       " '1StochasticProcesses',\n",
       " '2StochasticProcesses',\n",
       " '3StochasticProcesses',\n",
       " '4StochasticProcesses',\n",
       " '5StochasticProcesses',\n",
       " '6StochasticProcesses',\n",
       " '7StochasticProcesses',\n",
       " '8StochasticProcesses',\n",
       " '9StochasticProcesses',\n",
       " '10StochasticProcesses',\n",
       " '11StochasticProcesses',\n",
       " '12StochasticProcesses',\n",
       " '13StochasticProcesses',\n",
       " '14StochasticProcesses',\n",
       " '15StochasticProcesses',\n",
       " '16StochasticProcesses',\n",
       " '17StochasticProcesses',\n",
       " '18StochasticProcesses',\n",
       " '19StochasticProcesses',\n",
       " '20StochasticProcesses',\n",
       " '21StochasticProcesses',\n",
       " '22StochasticProcesses',\n",
       " '23StochasticProcesses',\n",
       " '24StochasticProcesses',\n",
       " '25StochasticProcesses',\n",
       " '26StochasticProcesses',\n",
       " '27StochasticProcesses',\n",
       " '28StochasticProcesses',\n",
       " '29StochasticProcesses',\n",
       " '30StochasticProcesses',\n",
       " '31StochasticProcesses',\n",
       " '32StochasticProcesses',\n",
       " '33StochasticProcesses',\n",
       " '34StochasticProcesses',\n",
       " '35StochasticProcesses',\n",
       " '36StochasticProcesses',\n",
       " '37StochasticProcesses',\n",
       " '38StochasticProcesses',\n",
       " '39StochasticProcesses',\n",
       " '40StochasticProcesses',\n",
       " '41StochasticProcesses',\n",
       " '42StochasticProcesses',\n",
       " '1ReinforcementLearning',\n",
       " '2ReinforcementLearning',\n",
       " '3ReinforcementLearning',\n",
       " '4ReinforcementLearning',\n",
       " '5ReinforcementLearning',\n",
       " '6ReinforcementLearning',\n",
       " '1ConvexOptimizationbook',\n",
       " '2ConvexOptimizationbook',\n",
       " '3ConvexOptimizationbook',\n",
       " '4ConvexOptimizationbook',\n",
       " '5ConvexOptimizationbook',\n",
       " '6ConvexOptimizationbook',\n",
       " '7ConvexOptimizationbook',\n",
       " '8ConvexOptimizationbook',\n",
       " '9ConvexOptimizationbook',\n",
       " '10ConvexOptimizationbook',\n",
       " '11ConvexOptimizationbook',\n",
       " '1RegressionModelingStrategies',\n",
       " '2RegressionModelingStrategies',\n",
       " '3RegressionModelingStrategies',\n",
       " '4RegressionModelingStrategies',\n",
       " '5RegressionModelingStrategies',\n",
       " '6RegressionModelingStrategies',\n",
       " '7RegressionModelingStrategies',\n",
       " '8RegressionModelingStrategies',\n",
       " '9RegressionModelingStrategies',\n",
       " '10RegressionModelingStrategies',\n",
       " '11RegressionModelingStrategies',\n",
       " '12RegressionModelingStrategies',\n",
       " '13RegressionModelingStrategies',\n",
       " '14RegressionModelingStrategies',\n",
       " '15RegressionModelingStrategies',\n",
       " '16RegressionModelingStrategies',\n",
       " '17RegressionModelingStrategies',\n",
       " '18RegressionModelingStrategies',\n",
       " '19RegressionModelingStrategies',\n",
       " '20RegressionModelingStrategies',\n",
       " '21RegressionModelingStrategies',\n",
       " '1ComputationalStatistics',\n",
       " '2ComputationalStatistics',\n",
       " '3ComputationalStatistics',\n",
       " '4ComputationalStatistics',\n",
       " '5ComputationalStatistics',\n",
       " '6ComputationalStatistics',\n",
       " '7ComputationalStatistics',\n",
       " '8ComputationalStatistics',\n",
       " '9ComputationalStatistics',\n",
       " '10ComputationalStatistics',\n",
       " '11ComputationalStatistics',\n",
       " '12ComputationalStatistics',\n",
       " '1RossProbability',\n",
       " '2RossProbability',\n",
       " '3RossProbability',\n",
       " '4RossProbability',\n",
       " '5RossProbability',\n",
       " '6RossProbability',\n",
       " '7RossProbability',\n",
       " '8RossProbability',\n",
       " '9RossProbability',\n",
       " '10RossProbability',\n",
       " '1CategoricalDataAnalysis',\n",
       " '2CategoricalDataAnalysis',\n",
       " '3CategoricalDataAnalysis',\n",
       " '4CategoricalDataAnalysis',\n",
       " '5CategoricalDataAnalysis',\n",
       " '6CategoricalDataAnalysis',\n",
       " '7CategoricalDataAnalysis',\n",
       " '8CategoricalDataAnalysis',\n",
       " '9CategoricalDataAnalysis',\n",
       " '10CategoricalDataAnalysis',\n",
       " '11CategoricalDataAnalysis',\n",
       " '12CategoricalDataAnalysis',\n",
       " '13CategoricalDataAnalysis',\n",
       " '14CategoricalDataAnalysis',\n",
       " '15CategoricalDataAnalysis',\n",
       " '16CategoricalDataAnalysis',\n",
       " '1DeepLearning',\n",
       " '2DeepLearning',\n",
       " '3DeepLearning',\n",
       " '4DeepLearning',\n",
       " '5DeepLearning',\n",
       " '6DeepLearning',\n",
       " '7DeepLearning',\n",
       " '8DeepLearning',\n",
       " '9DeepLearning',\n",
       " '10DeepLearning',\n",
       " '11DeepLearning',\n",
       " '12DeepLearning',\n",
       " '13DeepLearning',\n",
       " '14DeepLearning',\n",
       " '15DeepLearning',\n",
       " '16DeepLearning',\n",
       " '17DeepLearning',\n",
       " '18DeepLearning',\n",
       " '19DeepLearning',\n",
       " '20DeepLearning',\n",
       " '11L2RM',\n",
       " '12ConfidenceRegionLowRankRegre',\n",
       " '13PRCA']"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "chp_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "from evaluation import get_concept_pairs\n",
    "top_n_pairs = get_concept_pairs(incre_A, concept, n=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "conditional,old\n",
      "reinforcement learning,old\n",
      "gradient,old\n",
      "error,old\n",
      "odd,old\n",
      "likelihood,old\n",
      "probability,old\n",
      "mean,old\n",
      "response,old\n",
      "matrix,old\n",
      "poisson,old\n",
      "factor,old\n",
      "sample,old\n",
      "bounded,old\n",
      "regression,old\n",
      "variational inference,new\n",
      "local martingale,old\n",
      "limit theorem,old\n",
      "strong markov proces,old\n",
      "stability,new\n",
      "predictor,old\n",
      "fitting,old\n",
      "martingale problem,old\n",
      "markov chain,old\n",
      "continuou,old\n",
      "predictable,old\n",
      "itˆo’s formula,old\n",
      "cauchy,old\n",
      "residual,old\n",
      "variance,old\n",
      "q-function,old\n",
      "semigroup,old\n",
      "policy,old\n",
      "markov property,old\n",
      "optimization,old\n",
      "brownian motion,old\n",
      "expected,old\n",
      "estimator,old\n",
      "martingale,old\n",
      "variable,old\n",
      "set,new\n",
      "markov proces,old\n",
      "dirichlet,old\n",
      "parameter,old\n",
      "chain,old\n",
      "domain,old\n",
      "random variable,old\n",
      "norm,old\n",
      "natural scale,old\n",
      "filtration,old\n",
      "linear,old\n",
      "stopping time,old\n",
      "integral,old\n",
      "distribution,old\n",
      "event,old\n"
     ]
    }
   ],
   "source": [
    "top_concepts = set()\n",
    "for i in top_n_pairs:\n",
    "    top_concepts = top_concepts |set([i[0],i[1]])\n",
    "for i in top_concepts:\n",
    "    if concept.index(i) < len(concept) - 137:\n",
    "        print('{},old'.format(i))\n",
    "    else:\n",
    "        print('{},new'.format(i))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['predictor', 'predictor', 42.0289],\n",
       " ['distribution', 'likelihood', 33.2616],\n",
       " ['martingale', 'markov proces', 31.921],\n",
       " ['variable', 'sample', 31.1246],\n",
       " ['distribution', 'chain', 28.9517],\n",
       " ['regression', 'predictor', 28.9328],\n",
       " ['brownian motion', 'bounded', 28.7399],\n",
       " ['distribution', 'distribution', 27.8493],\n",
       " ['continuou', 'martingale problem', 27.427],\n",
       " ['brownian motion', 'martingale problem', 27.1609],\n",
       " ['variable', 'variable', 27.1169],\n",
       " ['bounded', 'probability', 27.1137],\n",
       " ['random variable', 'continuou', 26.9532],\n",
       " ['predictor', 'regression', 26.7117],\n",
       " ['random variable', 'sample', 26.3094],\n",
       " ['bounded', 'markov proces', 25.5463],\n",
       " ['policy', 'policy', 25.2993],\n",
       " ['predictor', 'response', 24.8257],\n",
       " ['probability', 'likelihood', 23.0761],\n",
       " ['brownian motion', 'q-function', 22.8027],\n",
       " ['likelihood', 'likelihood', 22.666],\n",
       " ['gradient', 'distribution', 21.6943],\n",
       " ['variable', 'regression', 21.3116],\n",
       " ['distribution', 'variance', 21.1618],\n",
       " ['bounded', 'martingale problem', 21.103],\n",
       " ['continuou', 'chain', 20.3639],\n",
       " ['regression', 'response', 20.1244],\n",
       " ['regression', 'regression', 20.082],\n",
       " ['random variable', 'brownian motion', 19.9224],\n",
       " ['poisson', 'bounded', 19.7817],\n",
       " ['markov proces', 'probability', 19.5847],\n",
       " ['bounded', 'semigroup', 19.4264],\n",
       " ['brownian motion', 'continuou', 19.1896],\n",
       " ['chain', 'distribution', 18.4047],\n",
       " ['markov proces', 'distribution', 18.318],\n",
       " ['probability', 'conditional', 17.818],\n",
       " ['martingale', 'martingale problem', 17.805],\n",
       " ['continuou', 'q-function', 17.4352],\n",
       " ['brownian motion', 'predictable', 17.3841],\n",
       " ['variational inference', 'fitting', 17.3571],\n",
       " ['bounded', 'chain', 17.3084],\n",
       " ['random variable', 'variable', 17.27],\n",
       " ['factor', 'predictor', 17.2106],\n",
       " ['gradient', 'gradient', 17.0263],\n",
       " ['sample', 'likelihood', 16.9988],\n",
       " ['probability', 'error', 16.9722],\n",
       " ['response', 'response', 16.85],\n",
       " ['reinforcement learning', 'stability', 16.8184],\n",
       " ['stopping time', 'continuou', 16.7051],\n",
       " ['event', 'predictor', 16.6955],\n",
       " ['distribution', 'markov chain', 16.611],\n",
       " ['martingale', 'markov property', 16.1283],\n",
       " ['event', 'regression', 15.9145],\n",
       " ['matrix', 'gradient', 15.8627],\n",
       " ['brownian motion', 'chain', 15.7758],\n",
       " ['brownian motion', 'reinforcement learning', 15.7402],\n",
       " ['random variable', 'estimator', 15.7005],\n",
       " ['variable', 'estimator', 15.6894],\n",
       " ['martingale', 'martingale', 15.4395],\n",
       " ['continuou', 'bounded', 15.3976],\n",
       " ['event', 'martingale', 15.258],\n",
       " ['brownian motion', 'random variable', 15.2358],\n",
       " ['variable', 'linear', 15.1986],\n",
       " ['bounded', 'dirichlet', 15.0767],\n",
       " ['parameter', 'martingale', 14.926],\n",
       " ['predictor', 'variable', 14.8064],\n",
       " ['integral', 'markov proces', 14.793],\n",
       " ['predictor', 'residual', 14.757],\n",
       " ['brownian motion', 'semigroup', 14.7403],\n",
       " ['brownian motion', 'local martingale', 14.6185],\n",
       " ['martingale', 'strong markov proces', 14.5209],\n",
       " ['filtration', 'martingale', 14.5084],\n",
       " ['expected', 'brownian motion', 14.4443],\n",
       " ['brownian motion', 'poisson', 14.4414],\n",
       " ['itˆo’s formula', 'markov proces', 14.3562],\n",
       " ['matrix', 'mean', 14.2243],\n",
       " ['continuou', 'cauchy', 14.0664],\n",
       " ['norm', 'gradient', 14.0574],\n",
       " ['variable', 'error', 14.0187],\n",
       " ['matrix', 'linear', 13.971],\n",
       " ['distribution', 'estimator', 13.9234],\n",
       " ['brownian motion', 'stopping time', 13.9048],\n",
       " ['limit theorem', 'probability', 13.8963],\n",
       " ['markov proces', 'matrix', 13.8837],\n",
       " ['brownian motion', 'natural scale', 13.6748],\n",
       " ['matrix', 'optimization', 13.6511],\n",
       " ['brownian motion', 'martingale', 13.6452],\n",
       " ['factor', 'variable', 13.6185],\n",
       " ['variable', 'odd', 13.6047],\n",
       " ['linear', 'set', 13.5269],\n",
       " ['bounded', 'domain', 13.52],\n",
       " ['bounded', 'markov property', 13.4803],\n",
       " ['set', 'policy', 13.4582],\n",
       " ['response', 'predictor', 13.4199],\n",
       " ['mean', 'probability', 13.4172],\n",
       " ['chain', 'chain', 13.3895],\n",
       " ['stopping time', 'martingale', 13.3514],\n",
       " ['optimization', 'gradient', 13.2888],\n",
       " ['martingale', 'variable', 13.274],\n",
       " ['predictor', 'factor', 13.2544]]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top_n_pairs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}