CiteRankDBLPv11_v1.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "import json\n",
    "import numpy as np\n",
    "from timeit import default_timer as timer\n",
    "\n",
    "from dblputils import IOUtil, CitationDataset\n",
    "from dblpfeatures import DataUtil\n",
    "from namedisambiguation_v1 import AuthorNameDisambiguation, MergeInfo"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Global variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Path names\n",
    "version = \"v11_reduced_300k\"\n",
    "LESS_COLS = \"./datasets/v11/dblp_papers_v11_less-cols.txt\"\n",
    "DATA_NAME = \"./datasets/\"+version+\"/dblp_papers_\"+version+\".txt\"\n",
    "AUTHORS = \"./datasets/\"+version+\"/dblp_authors_\"+version+\".txt\"\n",
    "# as 'AUTHORS', but after AuthorNameDisambiguation\n",
    "MERGED = \"./datasets/\"+version+\"/dblp_authors_merged_\"+version+\".txt\"\n",
    "\n",
    "# simple file which contains all the publication ids and the list of their authors\n",
    "CO_AUTH = \"./datasets/\"+version+\"/data_structures/pub_ids_auths.txt\"\n",
    "# simple file which contains all the author ids and names\n",
    "AUTHS_DICT = \"./datasets/\"+version+\"/data_structures/auths_dict.txt\"\n",
    "# simple file which contains all the years that appear in the dataset and an ordinal number that represent each of them\n",
    "YEARS_DICT = \"./datasets/\"+version+\"/data_structures/years_dict.txt\"\n",
    "# simple file which contains all the raw venues that appear in the dataset and an ordinal number that represent each of them\n",
    "VENUES_DICT = \"./datasets/\"+version+\"/data_structures/venues_dict.txt\"\n",
    "# list of titles translated in english and cleared with clear_text() (title i is referred to publication i)\n",
    "TITLES = \"./datasets/\"+version+\"/data_structures/titles.txt\"\n",
    "# simple file which contains the list of ids of all publications for each author\n",
    "PUBS = \"./datasets/\"+version+\"/data_structures/auth_pubs.txt\"\n",
    "\n",
    "nd_version = \"v1\"\n",
    "# contains a row for each author; each row is formed by an author's id and the list of the string-matching authors' ids\n",
    "STEP1_DUPL = \"./datasets/\"+version+\"/name_disambiguation/\"+nd_version+\"/step1_matches.txt\"\n",
    "# sparse matrix (author-publication) where a_ij=1 if the author i wrote the publication j, a_ij=0 otherwise\n",
    "STEP2_Map = \"./datasets/\"+version+\"/name_disambiguation/\"+nd_version+\"/step2_Map.npz\"\n",
    "# sparse matrix containing the similarity value based on meta-path APAPA for each pair of authors\n",
    "STEP2_Mapapa = \"./datasets/\"+version+\"/name_disambiguation/\"+nd_version+\"/step2_Mapapa.npz\"\n",
    "# sparse matrix containing the similarity value based on meta-path APYPA for each pair of authors\n",
    "STEP2_May = \"./datasets/\"+version+\"/name_disambiguation/\"+nd_version+\"/step2_May.npz\"\n",
    "# sparse matrix containing the similarity value based on meta-path APTPA for each pair of authors\n",
    "STEP2_Mat = \"./datasets/\"+version+\"/name_disambiguation/\"+nd_version+\"/step2_Mat.npz\"\n",
    "# sparse matrix containing the similarity value based on meta-path AVA for each pair of authors\n",
    "STEP2_Mav = \"./datasets/\"+version+\"/name_disambiguation/\"+nd_version+\"/step2_Mav.npz\"\n",
    "# contains a row for each author that has at least one possible duplicate; each row is formed by an author's id and the list of \n",
    "# similarity values (one for each of his possible duplicates)\n",
    "STEP2_SIM = \"./datasets/\"+version+\"/name_disambiguation/\"+nd_version+\"/step2_similarities_uniform.txt\"\n",
    "# contains a row for each author with matched duplicates; each row is formed by an author's id and the list of the matched\n",
    "# authors' ids\n",
    "STEP3_DUPL = \"./datasets/\"+version+\"/name_disambiguation/\"+nd_version+\"/step3_duplicates.txt\"\n",
    "# dictionary where for each <key,value> pair the key is an author id from the complete authors dataset and the value\n",
    "# is the same id if he's no one duplicate or his duplicate's id if he's someone duplicate\n",
    "MERGED_IDMAP = \"./datasets/\"+version+\"/name_disambiguation/\"+nd_version+\"/step3_idmap.txt\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# All the fields of a json object loaded from the dataset\n",
    "all_columns = np.array([\"id\", \"title\", \"authors\", \"venue\", \"year\", \"keywords\", \"fos\", \"references\",\n",
    "                        \"n_citation\", \"page_start\", \"page_end\", \"doc_type\", \"lang\", \"publisher\",\n",
    "                        \"volume\", \"issue\", \"issn\", \"isbn\", \"doi\", \"pdf\", \"url\", \"abstract\", \"indexed_abstract\"])\n",
    "# Reduced number of fields of a json objed loaded from the dataset\n",
    "reduced_columns = np.array([\"id\", \"title\", \"authors\", \"venue\", \"year\", \"references\"])\n",
    "# Columns of the authors dataset\n",
    "author_columns = np.array([\"id\", \"name\", \"org\", \"pubs\"])\n",
    "# authors['pubs'] inner keys\n",
    "author_pub_columns = np.array([\"id\", \"title\", \"year\", \"venue\", \"references\"])\n",
    "# total number of entries in the original dataset\n",
    "dataset_lines = 1425148      #4107340\n",
    "# total number of entries in the authors dataset\n",
    "authors_lines = 1145383      #3655052"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Instances of some classes or methods\n",
    "ioutil = IOUtil()\n",
    "datautil = DataUtil()\n",
    "authorNameDisambiguation = AuthorNameDisambiguation(ioutil)\n",
    "mergeinfo = MergeInfo(ioutil)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Operations on the original dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "publications = CitationDataset(\"./datasets/v11/dblp_papers_v11.txt\", 4107340)\n",
    "# reducing the number of columns in the whole dataset\n",
    "ioutil.dumpLinesFromJson(publications, LESS_COLS, reduced_columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "publications = np.array(ioutil.loadAsJson(LESS_COLS))\n",
    "# selecting 300000 random publications from the dataset\n",
    "ioutil.selectRandomPro(publications, 300000, DATA_NAME)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "auth_keys = [\"name\", \"org\"]\n",
    "pub_keys = [\"title\", \"year\"]\n",
    "pub_nested = {\"venue\":\"raw\"}\n",
    "# Authors dict structure:\n",
    "# dict := {<author_id>:<author_info>}\n",
    "#   <author_id> := str\n",
    "#   <author_info> := {\"id\":str, \"name\":str, \"org\":str, \"pubs\":list<pub_info>}\n",
    "#     <pub_info> := {\"id\":str, \"title\":str, \"year\":str, \"venue\":str, \"references\":list<str>}\n",
    "# extracting a dataset representation which focuses on authors instead of publications\n",
    "publications = CitationDataset(DATA_NAME, dataset_lines)\n",
    "authors = ioutil.extractAuthorsDataset(publications, auth_keys, pub_keys, pub_nested)\n",
    "ioutil.dumpLinesFromJson(list(authors.values()), AUTHORS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "authors = CitationDataset(AUTHORS, authors_lines)\n",
    "publications = CitationDataset(DATA_NAME, dataset_lines)\n",
    "datautil.extractAuthors(authors, save=True, filename=AUTHS_DICT)\n",
    "datautil.extractCoAuthors(publications, save=True, filename=CO_AUTH)\n",
    "datautil.extractPublications(authors, save=True, filename=PUBS)\n",
    "datautil.extractYears(publications, save=True, filename=YEARS_DICT)\n",
    "datautil.extractVenues(publications, save=True, filename=VENUES_DICT)\n",
    "specials = datautil.extractTitles(publications)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# referring to \"300k\" dataset\n",
    "import langid\n",
    "from googletrans import Translator\n",
    "lang_predictor = langid.classify\n",
    "lang_translator = Translator()\n",
    "\n",
    "not_en = []\n",
    "for i in range(len(specials)):\n",
    "    if(lang_predictor(specials[i])[0]!='en'):\n",
    "        not_en.append(i)\n",
    "    if(i%10==0):\n",
    "        print(\"\\Classified: %d/%d\"%(i,len(specials)),end='',flush=True)\n",
    "print(\"\\Classified: %d/%d\"%(i+1,len(specials)))\n",
    "en = list(set(np.arange(len(specials)))-set(not_en))\n",
    "done = 0\n",
    "for i in en:\n",
    "    specials[i] = datautil.clear_title(specials[i])\n",
    "    if(done%100==0):\n",
    "        print(\"\\rCorrected: %d/%d\"%(done,len(en)),end='',flush=True)\n",
    "    done += 1\n",
    "print(\"\\rCorrected: %d/%d\"%(done,len(en)))\n",
    "done = 0\n",
    "for i in not_en:\n",
    "    if(done<3622):\n",
    "        done += 1\n",
    "        continue\n",
    "    specials[i] = datautil.clear_title(lang_translator.translate(specials[i]).text)\n",
    "    if(done%100==0):\n",
    "        print(\"\\rCorrected: %d/%d\"%(done,len(not_en)),end='',flush=True)\n",
    "    done += 1\n",
    "print(\"\\rCorrected: %d/%d\"%(done,len(not_en)))\n",
    "copied = 0\n",
    "for i in range(len(datautil.titles_list)):\n",
    "    if(datautil.titles_list[i]==\"+\"):\n",
    "        datautil.titles_list[i] = specials[copied]\n",
    "        copied += 1\n",
    "    if(i%10000==0):\n",
    "        print(\"\\rSecured: %d/%d\"%(i,len(datautil.titles_list)),end='',flush=True)\n",
    "print(\"\\rSecured: %d/%d\"%(i+1,len(datautil.titles_list)))\n",
    "with open(TITLES, 'w', encoding='utf-8') as file:\n",
    "    for i in range(len(datautil.titles_list)):\n",
    "        file.write(\"%s%s\"%(datautil.titles_list[i],(\"\" if i==len(datautil.titles_list)-1 else \"\\n\")))\n",
    "        if(i%10000==0):\n",
    "            print(\"\\rLines written: %d/%d\"%(i,len(datautil.titles_list)),end='',flush=True)\n",
    "    print(\"\\rLines written: %d/%d\"%(i+1,len(datautil.titles_list)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Read everything from here:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Reading all the dictionaries\n",
    "datautil.loadAttributes(authors=AUTHS_DICT, coAuthors=CO_AUTH, publications=PUBS, \n",
    "                        venues=VENUES_DICT, years=YEARS_DICT, titles=TITLES)\n",
    "# Preparing convenient data structures for the next part\n",
    "datautil.computeAuthorsInd()\n",
    "datautil.computePublicationsInd()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Name Disambiguation Problem\n",
    "src: Ranking-Based Name Matching for Author Disambiguation in Bibliographic Data<br>\n",
    "readaptation: 3 steps -> r-step, p-step, merge-step"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***r-step***: *Improving the recall; given an author ID, one should find as many potential duplicates as possible via String-based consideration.*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Step 1: for each author find all possible duplicates using RatcliffObershelp similarity as criteria. A list \n",
    "# of names is also stored: if a name is completely equal to one already seen (homonim), than the iteration is \n",
    "# avoided and the list of possible duplicates is copied from the homonim.\n",
    "authorNameDisambiguation.r_step(datautil.authors_dict, STEP1_DUPL, start=527540) # +11681 mins"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***p-step***: *Improving the precision; once finding potential duplicates for each candidate author name, a critical task is to infer the real author entity shared by one or more author IDs. In order to accomplish this task is used the idea of meta-paths [AVA (same venue), APAPA (co-authors of my co-authors), APTPA (title similarities), APYPA (same year)].*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Step 2.0: computing the adjacency matrix Map (author-publication) where a_ij=1 if the author i wrote the \n",
    "# publication j, a_ij=0 otherwise\n",
    "authorNameDisambiguation.computeMap(datautil.auth_id_ind, datautil.pub_id_ind, datautil.publications_dict, filename=STEP2_Map)\n",
    "# Step 2.1: computing meta-path APAPA for each author (applying l2-norm)\n",
    "authorNameDisambiguation.computeMapapa(filename=STEP2_Mapapa)\n",
    "# Step 2.2: computing meta-path APYPA for each author (applying l2-norm)\n",
    "authorNameDisambiguation.computeMay(CitationDataset(DATA_Name, dataset_lines), datautil.pub_id_ind, datautil.years_dict, \n",
    "                                    filename=STEP2_May)\n",
    "# Step 2.3: computing meta-path APTPA for each author (applying l2-norm)\n",
    "authorNameDisambiguation.computeMat(datautil.titles_list, filename=STEP2_Mat)\n",
    "# Step 2.4: computing meta-path AVA for each author (applying PathSim)\n",
    "authorNameDisambiguation.computeMav(CitationDataset(AUTHORS, authors_lines), datautil.venues_dict, datautil.auth_id_ind, \n",
    "                                    filename=STEP2_Mav)\n",
    "# Step 2.5: computing the sum of similarity value of each meta-path for each author that has at least one possible duplicate\n",
    "authorNameDisambiguation.p_step(np.array([1/4]*4), datautil.auth_id_ind, filename=STEP2_SIM)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# RUN FROM HERE\n",
    "authorNameDisambiguation.loadPossibleDuplicates(STEP1_DUPL)\n",
    "authorNameDisambiguation.loadMatrices(Mapapa=STEP2_Mapapa, May=STEP2_May, Mat=STEP2_Mat, Mav=STEP2_Mav)\n",
    "authorNameDisambiguation.p_step(np.array([1/4]*4), datautil.auth_id_ind, filename=STEP2_SIM)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***m-step***: *Merging the resulting duplicates, based on the similarity values obtained in the r-step and the p-step.*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Step 3: finding the actual duplicates for each author that passed the first step, i.e. the possible duplicate authors \n",
    "# with a similarity value (meta-path based) over a threshold\n",
    "authorNameDisambiguation.m_step(filename=STEP3_DUPL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# saving the dataset with the merged authors\n",
    "authors = ioutil.loadAsJson(AUTHORS)\n",
    "tmp = dict()\n",
    "for entry in authors:\n",
    "    tmp[entry['id']] = entry\n",
    "authors = tmp\n",
    "del tmp\n",
    "mergeinfo.computeMergeDataset(authors, authorNameDisambiguation.duplicates, filename=MERGED)\n",
    "mergeinfo.computeIdMap(CitationDataset(AUTHORS, authors_lines), authorNameDisambiguation.duplicates, filename=MERGED_IDMAP)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}