dump_data_ref_to_json.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>.container { width:100% !important; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import sys\n",
    "\n",
    "#Due to leftoevererrors in Nathan's python installation, some cleaning up occurs here\n",
    "#sys.path.append(\"./code/\")\n",
    "#sys.path.remove('/usr/local/lib/python2.7/site-packages') \n",
    "\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import copy\n",
    "import datetime\n",
    "import pickle\n",
    "import gzip\n",
    "import os,glob\n",
    "import time\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import operator\n",
    "import random\n",
    "from  scipy import stats\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "from pymongo import MongoClient\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "from IPython.core.display import display,HTML\n",
    "display(HTML(\"<style>.container { width:100% !important; }</style>\"))  # to make the notebook use the entire width of the browser\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 19.5 s, sys: 6.08 s, total: 25.6 s\n",
      "Wall time: 1min 59s\n",
      "done loading pickles (10848620, 21)\n",
      "done removing ref_UT = -1 instances: (8907763, 21)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>occurence</th>\n",
       "      <th>paper_UT</th>\n",
       "      <th>paper_char_pos</th>\n",
       "      <th>paper_word_pos</th>\n",
       "      <th>para_char_pos</th>\n",
       "      <th>para_char_total</th>\n",
       "      <th>para_index</th>\n",
       "      <th>para_word_pos</th>\n",
       "      <th>para_word_total</th>\n",
       "      <th>reference_UT</th>\n",
       "      <th>...</th>\n",
       "      <th>sect_char_pos</th>\n",
       "      <th>sect_char_total</th>\n",
       "      <th>sect_index</th>\n",
       "      <th>sect_word_pos</th>\n",
       "      <th>sect_word_total</th>\n",
       "      <th>section_title</th>\n",
       "      <th>section_title_alt</th>\n",
       "      <th>regex_sect_index</th>\n",
       "      <th>ref_datetime</th>\n",
       "      <th>plos_datetime</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>324515600133</td>\n",
       "      <td>139</td>\n",
       "      <td>21</td>\n",
       "      <td>139</td>\n",
       "      <td>820</td>\n",
       "      <td>0</td>\n",
       "      <td>21</td>\n",
       "      <td>130</td>\n",
       "      <td>A1995QY75100004</td>\n",
       "      <td>...</td>\n",
       "      <td>139</td>\n",
       "      <td>4029</td>\n",
       "      <td>0</td>\n",
       "      <td>21</td>\n",
       "      <td>658</td>\n",
       "      <td>Introduction</td>\n",
       "      <td>introduction</td>\n",
       "      <td>0</td>\n",
       "      <td>1995-05-15 00:00:00</td>\n",
       "      <td>2013-09-04 00:00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>324515600133</td>\n",
       "      <td>16829</td>\n",
       "      <td>2992</td>\n",
       "      <td>494</td>\n",
       "      <td>1208</td>\n",
       "      <td>24</td>\n",
       "      <td>76</td>\n",
       "      <td>188</td>\n",
       "      <td>A1995QY75100004</td>\n",
       "      <td>...</td>\n",
       "      <td>494</td>\n",
       "      <td>5398</td>\n",
       "      <td>3</td>\n",
       "      <td>76</td>\n",
       "      <td>864</td>\n",
       "      <td>Discussion</td>\n",
       "      <td>discussion</td>\n",
       "      <td>3</td>\n",
       "      <td>1995-05-15 00:00:00</td>\n",
       "      <td>2013-09-04 00:00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>324515600133</td>\n",
       "      <td>17257</td>\n",
       "      <td>3057</td>\n",
       "      <td>922</td>\n",
       "      <td>1208</td>\n",
       "      <td>24</td>\n",
       "      <td>141</td>\n",
       "      <td>188</td>\n",
       "      <td>A1995QY75100004</td>\n",
       "      <td>...</td>\n",
       "      <td>922</td>\n",
       "      <td>5398</td>\n",
       "      <td>3</td>\n",
       "      <td>141</td>\n",
       "      <td>864</td>\n",
       "      <td>Discussion</td>\n",
       "      <td>discussion</td>\n",
       "      <td>3</td>\n",
       "      <td>1995-05-15 00:00:00</td>\n",
       "      <td>2013-09-04 00:00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>324515600133</td>\n",
       "      <td>266</td>\n",
       "      <td>42</td>\n",
       "      <td>266</td>\n",
       "      <td>820</td>\n",
       "      <td>0</td>\n",
       "      <td>42</td>\n",
       "      <td>130</td>\n",
       "      <td>000222499800017</td>\n",
       "      <td>...</td>\n",
       "      <td>266</td>\n",
       "      <td>4029</td>\n",
       "      <td>0</td>\n",
       "      <td>42</td>\n",
       "      <td>658</td>\n",
       "      <td>Introduction</td>\n",
       "      <td>introduction</td>\n",
       "      <td>0</td>\n",
       "      <td>2004-06-15 00:00:00</td>\n",
       "      <td>2013-09-04 00:00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2</td>\n",
       "      <td>324515600133</td>\n",
       "      <td>444</td>\n",
       "      <td>73</td>\n",
       "      <td>444</td>\n",
       "      <td>820</td>\n",
       "      <td>0</td>\n",
       "      <td>73</td>\n",
       "      <td>130</td>\n",
       "      <td>000222499800017</td>\n",
       "      <td>...</td>\n",
       "      <td>444</td>\n",
       "      <td>4029</td>\n",
       "      <td>0</td>\n",
       "      <td>73</td>\n",
       "      <td>658</td>\n",
       "      <td>Introduction</td>\n",
       "      <td>introduction</td>\n",
       "      <td>0</td>\n",
       "      <td>2004-06-15 00:00:00</td>\n",
       "      <td>2013-09-04 00:00:00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   occurence      paper_UT  paper_char_pos  paper_word_pos  para_char_pos  \\\n",
       "0          1  324515600133             139              21            139   \n",
       "1          2  324515600133           16829            2992            494   \n",
       "2          3  324515600133           17257            3057            922   \n",
       "3          1  324515600133             266              42            266   \n",
       "4          2  324515600133             444              73            444   \n",
       "\n",
       "   para_char_total  para_index  para_word_pos  para_word_total  \\\n",
       "0              820           0             21              130   \n",
       "1             1208          24             76              188   \n",
       "2             1208          24            141              188   \n",
       "3              820           0             42              130   \n",
       "4              820           0             73              130   \n",
       "\n",
       "      reference_UT         ...           sect_char_pos  sect_char_total  \\\n",
       "0  A1995QY75100004         ...                     139             4029   \n",
       "1  A1995QY75100004         ...                     494             5398   \n",
       "2  A1995QY75100004         ...                     922             5398   \n",
       "3  000222499800017         ...                     266             4029   \n",
       "4  000222499800017         ...                     444             4029   \n",
       "\n",
       "   sect_index  sect_word_pos  sect_word_total  section_title  \\\n",
       "0           0             21              658   Introduction   \n",
       "1           3             76              864     Discussion   \n",
       "2           3            141              864     Discussion   \n",
       "3           0             42              658   Introduction   \n",
       "4           0             73              658   Introduction   \n",
       "\n",
       "  section_title_alt regex_sect_index         ref_datetime        plos_datetime  \n",
       "0      introduction                0  1995-05-15 00:00:00  2013-09-04 00:00:00  \n",
       "1        discussion                3  1995-05-15 00:00:00  2013-09-04 00:00:00  \n",
       "2        discussion                3  1995-05-15 00:00:00  2013-09-04 00:00:00  \n",
       "3      introduction                0  2004-06-15 00:00:00  2013-09-04 00:00:00  \n",
       "4      introduction                0  2004-06-15 00:00:00  2013-09-04 00:00:00  \n",
       "\n",
       "[5 rows x 21 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "%time df_citation = pickle.load(open('../data/ref_dataframe_min.pkl', 'rb'))   # citation_dataframe.pkl\n",
    "print (\"done loading pickles\", df_citation.shape)\n",
    "\n",
    "\n",
    "\n",
    "df_citation = df_citation[df_citation['reference_UT'] != '-1']   # i dont know why, but there are 7 occurrences with value -1\n",
    "\n",
    "\n",
    "print (\"done removing ref_UT = -1 instances:\", df_citation.shape)\n",
    "\n",
    "\n",
    "df_citation.head()\n",
    "\n",
    "\n",
    "# lista_ref_UTs = list(df_citation.reference_UT.unique() )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>WoS_ID_reference</th>\n",
       "      <th>WoS_ID_plos</th>\n",
       "      <th>section</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A1995QY75100004</td>\n",
       "      <td>324515600133</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>A1995QY75100004</td>\n",
       "      <td>324515600133</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>000222499800017</td>\n",
       "      <td>324515600133</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>000222499800017</td>\n",
       "      <td>324515600133</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>000263911400006</td>\n",
       "      <td>324515600133</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>000289279600018</td>\n",
       "      <td>324515600133</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>000289279600018</td>\n",
       "      <td>324515600133</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>000287717700023</td>\n",
       "      <td>324515600133</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>000257589600013</td>\n",
       "      <td>324515600133</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>000257589600013</td>\n",
       "      <td>324515600133</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>000246465200001</td>\n",
       "      <td>324515600133</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>000246465200001</td>\n",
       "      <td>324515600133</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>000238112100022</td>\n",
       "      <td>324515600133</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>000266227800006</td>\n",
       "      <td>324515600133</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>000266227800006</td>\n",
       "      <td>324515600133</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>000220879700004</td>\n",
       "      <td>324515600133</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>000177105200006</td>\n",
       "      <td>324515600133</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>000182540700009</td>\n",
       "      <td>324515600133</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>000227197300045</td>\n",
       "      <td>324515600133</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>000299607700013</td>\n",
       "      <td>324515600133</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>000290020400009</td>\n",
       "      <td>324515600133</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>000290020400009</td>\n",
       "      <td>324515600133</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>A1994NV95600001</td>\n",
       "      <td>324515600133</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>A1994NV95600001</td>\n",
       "      <td>324515600133</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>000256313500005</td>\n",
       "      <td>324515600133</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>000293136200014</td>\n",
       "      <td>324515600133</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>000174597100019</td>\n",
       "      <td>324515600133</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>000288797100011</td>\n",
       "      <td>324515600133</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>000179315000003</td>\n",
       "      <td>324515600133</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>000179315000003</td>\n",
       "      <td>324515600133</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930646</th>\n",
       "      <td>000088538000012</td>\n",
       "      <td>321341000076</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930647</th>\n",
       "      <td>A1995RF14900014</td>\n",
       "      <td>321341000076</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930648</th>\n",
       "      <td>000167525500001</td>\n",
       "      <td>321341000076</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930649</th>\n",
       "      <td>A1992HZ04600007</td>\n",
       "      <td>321341000076</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930650</th>\n",
       "      <td>000268812400008</td>\n",
       "      <td>321341000076</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930651</th>\n",
       "      <td>000268812400008</td>\n",
       "      <td>321341000076</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930652</th>\n",
       "      <td>000285747900006</td>\n",
       "      <td>321341000076</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930653</th>\n",
       "      <td>000285747900006</td>\n",
       "      <td>321341000076</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930654</th>\n",
       "      <td>000090117600015</td>\n",
       "      <td>321341000076</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930655</th>\n",
       "      <td>000177854700008</td>\n",
       "      <td>321341000076</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930656</th>\n",
       "      <td>000247935300010</td>\n",
       "      <td>321341000076</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930657</th>\n",
       "      <td>A1982PW56700007</td>\n",
       "      <td>321341000076</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930658</th>\n",
       "      <td>A1980KX98200001</td>\n",
       "      <td>321341000076</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930659</th>\n",
       "      <td>000301971900007</td>\n",
       "      <td>321341000076</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930660</th>\n",
       "      <td>000264005500007</td>\n",
       "      <td>321341000076</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930661</th>\n",
       "      <td>000080163800019</td>\n",
       "      <td>321341000076</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930662</th>\n",
       "      <td>000185745000010</td>\n",
       "      <td>321341000076</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930663</th>\n",
       "      <td>A1982PH38400012</td>\n",
       "      <td>321341000076</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930664</th>\n",
       "      <td>A1989AE93800008</td>\n",
       "      <td>321341000076</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930665</th>\n",
       "      <td>000173977100041</td>\n",
       "      <td>321341000076</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930666</th>\n",
       "      <td>000175801600018</td>\n",
       "      <td>321341000076</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930667</th>\n",
       "      <td>000220747700018</td>\n",
       "      <td>321341000076</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930668</th>\n",
       "      <td>000174377700007</td>\n",
       "      <td>321341000076</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930669</th>\n",
       "      <td>000271586700013</td>\n",
       "      <td>321341000076</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930670</th>\n",
       "      <td>000221871800012</td>\n",
       "      <td>321341000076</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930671</th>\n",
       "      <td>A1954UF73200026</td>\n",
       "      <td>321341000076</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930672</th>\n",
       "      <td>000176187500027</td>\n",
       "      <td>321341000076</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930673</th>\n",
       "      <td>A1993LX59200002</td>\n",
       "      <td>321341000076</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930674</th>\n",
       "      <td>000183262600008</td>\n",
       "      <td>321341000076</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930675</th>\n",
       "      <td>000321228000014</td>\n",
       "      <td>321341000076</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>6930676 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        WoS_ID_reference   WoS_ID_plos  section\n",
       "0        A1995QY75100004  324515600133        0\n",
       "1        A1995QY75100004  324515600133        3\n",
       "2        000222499800017  324515600133        0\n",
       "3        000222499800017  324515600133        3\n",
       "4        000263911400006  324515600133        0\n",
       "5        000289279600018  324515600133        0\n",
       "6        000289279600018  324515600133        3\n",
       "7        000287717700023  324515600133        0\n",
       "8        000257589600013  324515600133        0\n",
       "9        000257589600013  324515600133        3\n",
       "10       000246465200001  324515600133        0\n",
       "11       000246465200001  324515600133        3\n",
       "12       000238112100022  324515600133        0\n",
       "13       000266227800006  324515600133        0\n",
       "14       000266227800006  324515600133        3\n",
       "15       000220879700004  324515600133        0\n",
       "16       000177105200006  324515600133        0\n",
       "17       000182540700009  324515600133        0\n",
       "18       000227197300045  324515600133        0\n",
       "19       000299607700013  324515600133        0\n",
       "20       000290020400009  324515600133        0\n",
       "21       000290020400009  324515600133        3\n",
       "22       A1994NV95600001  324515600133        0\n",
       "23       A1994NV95600001  324515600133        3\n",
       "24       000256313500005  324515600133        0\n",
       "25       000293136200014  324515600133        0\n",
       "26       000174597100019  324515600133        1\n",
       "27       000288797100011  324515600133        2\n",
       "28       000179315000003  324515600133        2\n",
       "29       000179315000003  324515600133        3\n",
       "...                  ...           ...      ...\n",
       "6930646  000088538000012  321341000076        0\n",
       "6930647  A1995RF14900014  321341000076        0\n",
       "6930648  000167525500001  321341000076        3\n",
       "6930649  A1992HZ04600007  321341000076        0\n",
       "6930650  000268812400008  321341000076        0\n",
       "6930651  000268812400008  321341000076        3\n",
       "6930652  000285747900006  321341000076        1\n",
       "6930653  000285747900006  321341000076        3\n",
       "6930654  000090117600015  321341000076        3\n",
       "6930655  000177854700008  321341000076        3\n",
       "6930656  000247935300010  321341000076        3\n",
       "6930657  A1982PW56700007  321341000076        3\n",
       "6930658  A1980KX98200001  321341000076        3\n",
       "6930659  000301971900007  321341000076        3\n",
       "6930660  000264005500007  321341000076        3\n",
       "6930661  000080163800019  321341000076        3\n",
       "6930662  000185745000010  321341000076        3\n",
       "6930663  A1982PH38400012  321341000076        3\n",
       "6930664  A1989AE93800008  321341000076        3\n",
       "6930665  000173977100041  321341000076        3\n",
       "6930666  000175801600018  321341000076        3\n",
       "6930667  000220747700018  321341000076        3\n",
       "6930668  000174377700007  321341000076        3\n",
       "6930669  000271586700013  321341000076        3\n",
       "6930670  000221871800012  321341000076        3\n",
       "6930671  A1954UF73200026  321341000076        3\n",
       "6930672  000176187500027  321341000076        3\n",
       "6930673  A1993LX59200002  321341000076        3\n",
       "6930674  000183262600008  321341000076        3\n",
       "6930675  000321228000014  321341000076        3\n",
       "\n",
       "[6930676 rows x 3 columns]"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "##### i simplify the file, dropping all columns i dont need, dropping duplicates and reindexing df\n",
    "\n",
    "len(df_citation.reference_UT.unique() ) # 2,607,457\n",
    "\n",
    "len(df_citation.paper_UT.unique() ) # 158,813\n",
    "\n",
    "\n",
    "df_ref_one_per_sect = df_citation.drop_duplicates(subset=['reference_UT', 'paper_UT','sect_index'])\n",
    "df_ref_one_per_sect = df_ref_one_per_sect[['reference_UT', 'paper_UT','sect_index']] \n",
    "\n",
    "\n",
    "df_ref_one_per_sect = df_ref_one_per_sect.rename(columns={'sect_index': 'section', 'paper_UT': 'WoS_ID_plos', 'reference_UT': 'WoS_ID_reference'})\n",
    "\n",
    "df_ref_one_per_sect.shape    # 6,930,676\n",
    "\n",
    "\n",
    "df_ref_one_per_sect.reset_index(drop=True, inplace=True)\n",
    "df_ref_one_per_sect\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1min 35s, sys: 249 ms, total: 1min 35s\n",
      "Wall time: 1min 37s\n"
     ]
    }
   ],
   "source": [
    "### i fix the issue of the missing 000 in the paper_UTs\n",
    "\n",
    "\n",
    "def get_new_paper_UT(row):\n",
    "    \n",
    "    new_UT= '000'+str(row.WoS_ID_plos)\n",
    "   \n",
    "    return new_UT\n",
    "\n",
    "###################\n",
    "\n",
    "\n",
    "%time df_ref_one_per_sect['new_paper_UT'] = df_ref_one_per_sect.apply (lambda row: get_new_paper_UT(row),axis=1)\n",
    "\n",
    "df_ref_one_per_sect.drop(['WoS_ID_plos'], axis=1, inplace=True)\n",
    "df_ref_one_per_sect.rename(columns={'new_paper_UT': 'WoS_ID_plos'}, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "WoS_ID_reference    object\n",
       "section              int64\n",
       "WoS_ID_plos         object\n",
       "dtype: object"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_ref_one_per_sect.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>WoS_ID_reference</th>\n",
       "      <th>section</th>\n",
       "      <th>WoS_ID_plos</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A1995QY75100004</td>\n",
       "      <td>0</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>A1995QY75100004</td>\n",
       "      <td>3</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>000222499800017</td>\n",
       "      <td>0</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>000222499800017</td>\n",
       "      <td>3</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>000263911400006</td>\n",
       "      <td>0</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>000289279600018</td>\n",
       "      <td>0</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>000289279600018</td>\n",
       "      <td>3</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>000287717700023</td>\n",
       "      <td>0</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>000257589600013</td>\n",
       "      <td>0</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>000257589600013</td>\n",
       "      <td>3</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>000246465200001</td>\n",
       "      <td>0</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>000246465200001</td>\n",
       "      <td>3</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>000238112100022</td>\n",
       "      <td>0</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>000266227800006</td>\n",
       "      <td>0</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>000266227800006</td>\n",
       "      <td>3</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>000220879700004</td>\n",
       "      <td>0</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>000177105200006</td>\n",
       "      <td>0</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>000182540700009</td>\n",
       "      <td>0</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>000227197300045</td>\n",
       "      <td>0</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>000299607700013</td>\n",
       "      <td>0</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>000290020400009</td>\n",
       "      <td>0</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>000290020400009</td>\n",
       "      <td>3</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>A1994NV95600001</td>\n",
       "      <td>0</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>A1994NV95600001</td>\n",
       "      <td>3</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>000256313500005</td>\n",
       "      <td>0</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>000293136200014</td>\n",
       "      <td>0</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>000174597100019</td>\n",
       "      <td>1</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>000288797100011</td>\n",
       "      <td>2</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>000179315000003</td>\n",
       "      <td>2</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>000179315000003</td>\n",
       "      <td>3</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930646</th>\n",
       "      <td>000088538000012</td>\n",
       "      <td>0</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930647</th>\n",
       "      <td>A1995RF14900014</td>\n",
       "      <td>0</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930648</th>\n",
       "      <td>000167525500001</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930649</th>\n",
       "      <td>A1992HZ04600007</td>\n",
       "      <td>0</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930650</th>\n",
       "      <td>000268812400008</td>\n",
       "      <td>0</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930651</th>\n",
       "      <td>000268812400008</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930652</th>\n",
       "      <td>000285747900006</td>\n",
       "      <td>1</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930653</th>\n",
       "      <td>000285747900006</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930654</th>\n",
       "      <td>000090117600015</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930655</th>\n",
       "      <td>000177854700008</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930656</th>\n",
       "      <td>000247935300010</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930657</th>\n",
       "      <td>A1982PW56700007</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930658</th>\n",
       "      <td>A1980KX98200001</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930659</th>\n",
       "      <td>000301971900007</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930660</th>\n",
       "      <td>000264005500007</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930661</th>\n",
       "      <td>000080163800019</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930662</th>\n",
       "      <td>000185745000010</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930663</th>\n",
       "      <td>A1982PH38400012</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930664</th>\n",
       "      <td>A1989AE93800008</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930665</th>\n",
       "      <td>000173977100041</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930666</th>\n",
       "      <td>000175801600018</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930667</th>\n",
       "      <td>000220747700018</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930668</th>\n",
       "      <td>000174377700007</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930669</th>\n",
       "      <td>000271586700013</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930670</th>\n",
       "      <td>000221871800012</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930671</th>\n",
       "      <td>A1954UF73200026</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930672</th>\n",
       "      <td>000176187500027</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930673</th>\n",
       "      <td>A1993LX59200002</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930674</th>\n",
       "      <td>000183262600008</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930675</th>\n",
       "      <td>000321228000014</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>6930676 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        WoS_ID_reference  section      WoS_ID_plos\n",
       "0        A1995QY75100004        0  000324515600133\n",
       "1        A1995QY75100004        3  000324515600133\n",
       "2        000222499800017        0  000324515600133\n",
       "3        000222499800017        3  000324515600133\n",
       "4        000263911400006        0  000324515600133\n",
       "5        000289279600018        0  000324515600133\n",
       "6        000289279600018        3  000324515600133\n",
       "7        000287717700023        0  000324515600133\n",
       "8        000257589600013        0  000324515600133\n",
       "9        000257589600013        3  000324515600133\n",
       "10       000246465200001        0  000324515600133\n",
       "11       000246465200001        3  000324515600133\n",
       "12       000238112100022        0  000324515600133\n",
       "13       000266227800006        0  000324515600133\n",
       "14       000266227800006        3  000324515600133\n",
       "15       000220879700004        0  000324515600133\n",
       "16       000177105200006        0  000324515600133\n",
       "17       000182540700009        0  000324515600133\n",
       "18       000227197300045        0  000324515600133\n",
       "19       000299607700013        0  000324515600133\n",
       "20       000290020400009        0  000324515600133\n",
       "21       000290020400009        3  000324515600133\n",
       "22       A1994NV95600001        0  000324515600133\n",
       "23       A1994NV95600001        3  000324515600133\n",
       "24       000256313500005        0  000324515600133\n",
       "25       000293136200014        0  000324515600133\n",
       "26       000174597100019        1  000324515600133\n",
       "27       000288797100011        2  000324515600133\n",
       "28       000179315000003        2  000324515600133\n",
       "29       000179315000003        3  000324515600133\n",
       "...                  ...      ...              ...\n",
       "6930646  000088538000012        0  000321341000076\n",
       "6930647  A1995RF14900014        0  000321341000076\n",
       "6930648  000167525500001        3  000321341000076\n",
       "6930649  A1992HZ04600007        0  000321341000076\n",
       "6930650  000268812400008        0  000321341000076\n",
       "6930651  000268812400008        3  000321341000076\n",
       "6930652  000285747900006        1  000321341000076\n",
       "6930653  000285747900006        3  000321341000076\n",
       "6930654  000090117600015        3  000321341000076\n",
       "6930655  000177854700008        3  000321341000076\n",
       "6930656  000247935300010        3  000321341000076\n",
       "6930657  A1982PW56700007        3  000321341000076\n",
       "6930658  A1980KX98200001        3  000321341000076\n",
       "6930659  000301971900007        3  000321341000076\n",
       "6930660  000264005500007        3  000321341000076\n",
       "6930661  000080163800019        3  000321341000076\n",
       "6930662  000185745000010        3  000321341000076\n",
       "6930663  A1982PH38400012        3  000321341000076\n",
       "6930664  A1989AE93800008        3  000321341000076\n",
       "6930665  000173977100041        3  000321341000076\n",
       "6930666  000175801600018        3  000321341000076\n",
       "6930667  000220747700018        3  000321341000076\n",
       "6930668  000174377700007        3  000321341000076\n",
       "6930669  000271586700013        3  000321341000076\n",
       "6930670  000221871800012        3  000321341000076\n",
       "6930671  A1954UF73200026        3  000321341000076\n",
       "6930672  000176187500027        3  000321341000076\n",
       "6930673  A1993LX59200002        3  000321341000076\n",
       "6930674  000183262600008        3  000321341000076\n",
       "6930675  000321228000014        3  000321341000076\n",
       "\n",
       "[6930676 rows x 3 columns]"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_ref_one_per_sect"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "written: ../data/df_plos_WoS_id_refs_sects_for_public.pkl 6930676\n",
      "written:  ../data/df_plos_WoS_id_refs_sects_for_public.json\n",
      "written:  ../data/df_plos_WoS_id_refs_sects_for_public.csv\n"
     ]
    }
   ],
   "source": [
    "\n",
    "#### i print out the pandas dataframe\n",
    "with open('../data/df_plos_WoS_id_refs_sects_for_public.pkl', 'wb') as handle:\n",
    "    pickle.dump(df_ref_one_per_sect, handle, protocol = 2)\n",
    "print (\"written:\",'../data/df_plos_WoS_id_refs_sects_for_public.pkl',len(df_ref_one_per_sect))   \n",
    "\n",
    "\n",
    "\n",
    "#### create json files with basic info from the original dataframe, for sharing at publication\n",
    "df_ref_one_per_sect.to_json(path_or_buf = '../data/df_plos_WoS_id_refs_sects_for_public.json', orient= 'records')  # check out documentation for df.to_json()  # check out documentation for 'orient = split vs records'\n",
    "\n",
    "print ('written:  ../data/df_plos_WoS_id_refs_sects_for_public.json')\n",
    "\n",
    "\n",
    "\n",
    "df_ref_one_per_sect.to_csv(path_or_buf='../data/df_plos_WoS_id_refs_sects_for_public.csv', sep=';', na_rep='')\n",
    "print ('written:  ../data/df_plos_WoS_id_refs_sects_for_public.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'pd' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-93364a886842>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mtesting_df\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_json\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath_or_buf\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'../data/df_plos_WoS_id_refs_sects_for_public.json'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0morient\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'records'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'str'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mtesting_df\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'pd' is not defined"
     ]
    }
   ],
   "source": [
    "testing_df = pd.read_json(path_or_buf='../data/df_plos_WoS_id_refs_sects_for_public.json', orient='records',dtype='str')\n",
    "testing_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1.56 s, sys: 161 ms, total: 1.73 s\n",
      "Wall time: 3.13 s\n",
      "done loading pickles (6930676, 3)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>WoS_ID_reference</th>\n",
       "      <th>section</th>\n",
       "      <th>WoS_ID_plos</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A1995QY75100004</td>\n",
       "      <td>0</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>A1995QY75100004</td>\n",
       "      <td>3</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>000222499800017</td>\n",
       "      <td>0</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>000222499800017</td>\n",
       "      <td>3</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>000263911400006</td>\n",
       "      <td>0</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>000289279600018</td>\n",
       "      <td>0</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>000289279600018</td>\n",
       "      <td>3</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>000287717700023</td>\n",
       "      <td>0</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>000257589600013</td>\n",
       "      <td>0</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>000257589600013</td>\n",
       "      <td>3</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>000246465200001</td>\n",
       "      <td>0</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>000246465200001</td>\n",
       "      <td>3</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>000238112100022</td>\n",
       "      <td>0</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>000266227800006</td>\n",
       "      <td>0</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>000266227800006</td>\n",
       "      <td>3</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>000220879700004</td>\n",
       "      <td>0</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>000177105200006</td>\n",
       "      <td>0</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>000182540700009</td>\n",
       "      <td>0</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>000227197300045</td>\n",
       "      <td>0</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>000299607700013</td>\n",
       "      <td>0</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>000290020400009</td>\n",
       "      <td>0</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>000290020400009</td>\n",
       "      <td>3</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>A1994NV95600001</td>\n",
       "      <td>0</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>A1994NV95600001</td>\n",
       "      <td>3</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>000256313500005</td>\n",
       "      <td>0</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>000293136200014</td>\n",
       "      <td>0</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>000174597100019</td>\n",
       "      <td>1</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>000288797100011</td>\n",
       "      <td>2</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>000179315000003</td>\n",
       "      <td>2</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>000179315000003</td>\n",
       "      <td>3</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930646</th>\n",
       "      <td>000088538000012</td>\n",
       "      <td>0</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930647</th>\n",
       "      <td>A1995RF14900014</td>\n",
       "      <td>0</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930648</th>\n",
       "      <td>000167525500001</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930649</th>\n",
       "      <td>A1992HZ04600007</td>\n",
       "      <td>0</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930650</th>\n",
       "      <td>000268812400008</td>\n",
       "      <td>0</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930651</th>\n",
       "      <td>000268812400008</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930652</th>\n",
       "      <td>000285747900006</td>\n",
       "      <td>1</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930653</th>\n",
       "      <td>000285747900006</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930654</th>\n",
       "      <td>000090117600015</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930655</th>\n",
       "      <td>000177854700008</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930656</th>\n",
       "      <td>000247935300010</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930657</th>\n",
       "      <td>A1982PW56700007</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930658</th>\n",
       "      <td>A1980KX98200001</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930659</th>\n",
       "      <td>000301971900007</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930660</th>\n",
       "      <td>000264005500007</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930661</th>\n",
       "      <td>000080163800019</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930662</th>\n",
       "      <td>000185745000010</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930663</th>\n",
       "      <td>A1982PH38400012</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930664</th>\n",
       "      <td>A1989AE93800008</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930665</th>\n",
       "      <td>000173977100041</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930666</th>\n",
       "      <td>000175801600018</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930667</th>\n",
       "      <td>000220747700018</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930668</th>\n",
       "      <td>000174377700007</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930669</th>\n",
       "      <td>000271586700013</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930670</th>\n",
       "      <td>000221871800012</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930671</th>\n",
       "      <td>A1954UF73200026</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930672</th>\n",
       "      <td>000176187500027</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930673</th>\n",
       "      <td>A1993LX59200002</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930674</th>\n",
       "      <td>000183262600008</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930675</th>\n",
       "      <td>000321228000014</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>6930676 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        WoS_ID_reference  section      WoS_ID_plos\n",
       "0        A1995QY75100004        0  000324515600133\n",
       "1        A1995QY75100004        3  000324515600133\n",
       "2        000222499800017        0  000324515600133\n",
       "3        000222499800017        3  000324515600133\n",
       "4        000263911400006        0  000324515600133\n",
       "5        000289279600018        0  000324515600133\n",
       "6        000289279600018        3  000324515600133\n",
       "7        000287717700023        0  000324515600133\n",
       "8        000257589600013        0  000324515600133\n",
       "9        000257589600013        3  000324515600133\n",
       "10       000246465200001        0  000324515600133\n",
       "11       000246465200001        3  000324515600133\n",
       "12       000238112100022        0  000324515600133\n",
       "13       000266227800006        0  000324515600133\n",
       "14       000266227800006        3  000324515600133\n",
       "15       000220879700004        0  000324515600133\n",
       "16       000177105200006        0  000324515600133\n",
       "17       000182540700009        0  000324515600133\n",
       "18       000227197300045        0  000324515600133\n",
       "19       000299607700013        0  000324515600133\n",
       "20       000290020400009        0  000324515600133\n",
       "21       000290020400009        3  000324515600133\n",
       "22       A1994NV95600001        0  000324515600133\n",
       "23       A1994NV95600001        3  000324515600133\n",
       "24       000256313500005        0  000324515600133\n",
       "25       000293136200014        0  000324515600133\n",
       "26       000174597100019        1  000324515600133\n",
       "27       000288797100011        2  000324515600133\n",
       "28       000179315000003        2  000324515600133\n",
       "29       000179315000003        3  000324515600133\n",
       "...                  ...      ...              ...\n",
       "6930646  000088538000012        0  000321341000076\n",
       "6930647  A1995RF14900014        0  000321341000076\n",
       "6930648  000167525500001        3  000321341000076\n",
       "6930649  A1992HZ04600007        0  000321341000076\n",
       "6930650  000268812400008        0  000321341000076\n",
       "6930651  000268812400008        3  000321341000076\n",
       "6930652  000285747900006        1  000321341000076\n",
       "6930653  000285747900006        3  000321341000076\n",
       "6930654  000090117600015        3  000321341000076\n",
       "6930655  000177854700008        3  000321341000076\n",
       "6930656  000247935300010        3  000321341000076\n",
       "6930657  A1982PW56700007        3  000321341000076\n",
       "6930658  A1980KX98200001        3  000321341000076\n",
       "6930659  000301971900007        3  000321341000076\n",
       "6930660  000264005500007        3  000321341000076\n",
       "6930661  000080163800019        3  000321341000076\n",
       "6930662  000185745000010        3  000321341000076\n",
       "6930663  A1982PH38400012        3  000321341000076\n",
       "6930664  A1989AE93800008        3  000321341000076\n",
       "6930665  000173977100041        3  000321341000076\n",
       "6930666  000175801600018        3  000321341000076\n",
       "6930667  000220747700018        3  000321341000076\n",
       "6930668  000174377700007        3  000321341000076\n",
       "6930669  000271586700013        3  000321341000076\n",
       "6930670  000221871800012        3  000321341000076\n",
       "6930671  A1954UF73200026        3  000321341000076\n",
       "6930672  000176187500027        3  000321341000076\n",
       "6930673  A1993LX59200002        3  000321341000076\n",
       "6930674  000183262600008        3  000321341000076\n",
       "6930675  000321228000014        3  000321341000076\n",
       "\n",
       "[6930676 rows x 3 columns]"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "%time testing_df2 = pickle.load(open('../data/df_plos_WoS_id_refs_sects_for_public.pkl', 'rb'))   # citation_dataframe.pkl\n",
    "print (\"done loading pickles\", testing_df2.shape)\n",
    "testing_df2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "WoS_ID_reference    object\n",
       "section              int64\n",
       "WoS_ID_plos         object\n",
       "dtype: object"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "testing_df2.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>WoS_ID_reference</th>\n",
       "      <th>section</th>\n",
       "      <th>WoS_ID_plos</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A1995QY75100004</td>\n",
       "      <td>0</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>A1995QY75100004</td>\n",
       "      <td>3</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>000222499800017</td>\n",
       "      <td>0</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>000222499800017</td>\n",
       "      <td>3</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>000263911400006</td>\n",
       "      <td>0</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>000289279600018</td>\n",
       "      <td>0</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>000289279600018</td>\n",
       "      <td>3</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>000287717700023</td>\n",
       "      <td>0</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>000257589600013</td>\n",
       "      <td>0</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>000257589600013</td>\n",
       "      <td>3</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>000246465200001</td>\n",
       "      <td>0</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>000246465200001</td>\n",
       "      <td>3</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>000238112100022</td>\n",
       "      <td>0</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>000266227800006</td>\n",
       "      <td>0</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>000266227800006</td>\n",
       "      <td>3</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>000220879700004</td>\n",
       "      <td>0</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>000177105200006</td>\n",
       "      <td>0</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>000182540700009</td>\n",
       "      <td>0</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>000227197300045</td>\n",
       "      <td>0</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>000299607700013</td>\n",
       "      <td>0</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>000290020400009</td>\n",
       "      <td>0</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>000290020400009</td>\n",
       "      <td>3</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>A1994NV95600001</td>\n",
       "      <td>0</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>A1994NV95600001</td>\n",
       "      <td>3</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>000256313500005</td>\n",
       "      <td>0</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>000293136200014</td>\n",
       "      <td>0</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>000174597100019</td>\n",
       "      <td>1</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>000288797100011</td>\n",
       "      <td>2</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>000179315000003</td>\n",
       "      <td>2</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>000179315000003</td>\n",
       "      <td>3</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930646</th>\n",
       "      <td>000088538000012</td>\n",
       "      <td>0</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930647</th>\n",
       "      <td>A1995RF14900014</td>\n",
       "      <td>0</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930648</th>\n",
       "      <td>000167525500001</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930649</th>\n",
       "      <td>A1992HZ04600007</td>\n",
       "      <td>0</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930650</th>\n",
       "      <td>000268812400008</td>\n",
       "      <td>0</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930651</th>\n",
       "      <td>000268812400008</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930652</th>\n",
       "      <td>000285747900006</td>\n",
       "      <td>1</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930653</th>\n",
       "      <td>000285747900006</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930654</th>\n",
       "      <td>000090117600015</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930655</th>\n",
       "      <td>000177854700008</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930656</th>\n",
       "      <td>000247935300010</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930657</th>\n",
       "      <td>A1982PW56700007</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930658</th>\n",
       "      <td>A1980KX98200001</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930659</th>\n",
       "      <td>000301971900007</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930660</th>\n",
       "      <td>000264005500007</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930661</th>\n",
       "      <td>000080163800019</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930662</th>\n",
       "      <td>000185745000010</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930663</th>\n",
       "      <td>A1982PH38400012</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930664</th>\n",
       "      <td>A1989AE93800008</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930665</th>\n",
       "      <td>000173977100041</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930666</th>\n",
       "      <td>000175801600018</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930667</th>\n",
       "      <td>000220747700018</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930668</th>\n",
       "      <td>000174377700007</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930669</th>\n",
       "      <td>000271586700013</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930670</th>\n",
       "      <td>000221871800012</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930671</th>\n",
       "      <td>A1954UF73200026</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930672</th>\n",
       "      <td>000176187500027</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930673</th>\n",
       "      <td>A1993LX59200002</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930674</th>\n",
       "      <td>000183262600008</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930675</th>\n",
       "      <td>000321228000014</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>6930676 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        WoS_ID_reference  section      WoS_ID_plos\n",
       "0        A1995QY75100004        0  000324515600133\n",
       "1        A1995QY75100004        3  000324515600133\n",
       "2        000222499800017        0  000324515600133\n",
       "3        000222499800017        3  000324515600133\n",
       "4        000263911400006        0  000324515600133\n",
       "5        000289279600018        0  000324515600133\n",
       "6        000289279600018        3  000324515600133\n",
       "7        000287717700023        0  000324515600133\n",
       "8        000257589600013        0  000324515600133\n",
       "9        000257589600013        3  000324515600133\n",
       "10       000246465200001        0  000324515600133\n",
       "11       000246465200001        3  000324515600133\n",
       "12       000238112100022        0  000324515600133\n",
       "13       000266227800006        0  000324515600133\n",
       "14       000266227800006        3  000324515600133\n",
       "15       000220879700004        0  000324515600133\n",
       "16       000177105200006        0  000324515600133\n",
       "17       000182540700009        0  000324515600133\n",
       "18       000227197300045        0  000324515600133\n",
       "19       000299607700013        0  000324515600133\n",
       "20       000290020400009        0  000324515600133\n",
       "21       000290020400009        3  000324515600133\n",
       "22       A1994NV95600001        0  000324515600133\n",
       "23       A1994NV95600001        3  000324515600133\n",
       "24       000256313500005        0  000324515600133\n",
       "25       000293136200014        0  000324515600133\n",
       "26       000174597100019        1  000324515600133\n",
       "27       000288797100011        2  000324515600133\n",
       "28       000179315000003        2  000324515600133\n",
       "29       000179315000003        3  000324515600133\n",
       "...                  ...      ...              ...\n",
       "6930646  000088538000012        0  000321341000076\n",
       "6930647  A1995RF14900014        0  000321341000076\n",
       "6930648  000167525500001        3  000321341000076\n",
       "6930649  A1992HZ04600007        0  000321341000076\n",
       "6930650  000268812400008        0  000321341000076\n",
       "6930651  000268812400008        3  000321341000076\n",
       "6930652  000285747900006        1  000321341000076\n",
       "6930653  000285747900006        3  000321341000076\n",
       "6930654  000090117600015        3  000321341000076\n",
       "6930655  000177854700008        3  000321341000076\n",
       "6930656  000247935300010        3  000321341000076\n",
       "6930657  A1982PW56700007        3  000321341000076\n",
       "6930658  A1980KX98200001        3  000321341000076\n",
       "6930659  000301971900007        3  000321341000076\n",
       "6930660  000264005500007        3  000321341000076\n",
       "6930661  000080163800019        3  000321341000076\n",
       "6930662  000185745000010        3  000321341000076\n",
       "6930663  A1982PH38400012        3  000321341000076\n",
       "6930664  A1989AE93800008        3  000321341000076\n",
       "6930665  000173977100041        3  000321341000076\n",
       "6930666  000175801600018        3  000321341000076\n",
       "6930667  000220747700018        3  000321341000076\n",
       "6930668  000174377700007        3  000321341000076\n",
       "6930669  000271586700013        3  000321341000076\n",
       "6930670  000221871800012        3  000321341000076\n",
       "6930671  A1954UF73200026        3  000321341000076\n",
       "6930672  000176187500027        3  000321341000076\n",
       "6930673  A1993LX59200002        3  000321341000076\n",
       "6930674  000183262600008        3  000321341000076\n",
       "6930675  000321228000014        3  000321341000076\n",
       "\n",
       "[6930676 rows x 3 columns]"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_ref_one_per_sect"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "done loading pickles (6930676, 3)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>WoS_ID_reference</th>\n",
       "      <th>section</th>\n",
       "      <th>WoS_ID_plos</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A1995QY75100004</td>\n",
       "      <td>0</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>A1995QY75100004</td>\n",
       "      <td>3</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>000222499800017</td>\n",
       "      <td>0</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>000222499800017</td>\n",
       "      <td>3</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>000263911400006</td>\n",
       "      <td>0</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>000289279600018</td>\n",
       "      <td>0</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>000289279600018</td>\n",
       "      <td>3</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>000287717700023</td>\n",
       "      <td>0</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>000257589600013</td>\n",
       "      <td>0</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>000257589600013</td>\n",
       "      <td>3</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>000246465200001</td>\n",
       "      <td>0</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>000246465200001</td>\n",
       "      <td>3</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>000238112100022</td>\n",
       "      <td>0</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>000266227800006</td>\n",
       "      <td>0</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>000266227800006</td>\n",
       "      <td>3</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>000220879700004</td>\n",
       "      <td>0</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>000177105200006</td>\n",
       "      <td>0</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>000182540700009</td>\n",
       "      <td>0</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>000227197300045</td>\n",
       "      <td>0</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>000299607700013</td>\n",
       "      <td>0</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>000290020400009</td>\n",
       "      <td>0</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>000290020400009</td>\n",
       "      <td>3</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>A1994NV95600001</td>\n",
       "      <td>0</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>A1994NV95600001</td>\n",
       "      <td>3</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>000256313500005</td>\n",
       "      <td>0</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>000293136200014</td>\n",
       "      <td>0</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>000174597100019</td>\n",
       "      <td>1</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>000288797100011</td>\n",
       "      <td>2</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>000179315000003</td>\n",
       "      <td>2</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>000179315000003</td>\n",
       "      <td>3</td>\n",
       "      <td>000324515600133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930646</th>\n",
       "      <td>000088538000012</td>\n",
       "      <td>0</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930647</th>\n",
       "      <td>A1995RF14900014</td>\n",
       "      <td>0</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930648</th>\n",
       "      <td>000167525500001</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930649</th>\n",
       "      <td>A1992HZ04600007</td>\n",
       "      <td>0</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930650</th>\n",
       "      <td>000268812400008</td>\n",
       "      <td>0</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930651</th>\n",
       "      <td>000268812400008</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930652</th>\n",
       "      <td>000285747900006</td>\n",
       "      <td>1</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930653</th>\n",
       "      <td>000285747900006</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930654</th>\n",
       "      <td>000090117600015</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930655</th>\n",
       "      <td>000177854700008</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930656</th>\n",
       "      <td>000247935300010</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930657</th>\n",
       "      <td>A1982PW56700007</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930658</th>\n",
       "      <td>A1980KX98200001</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930659</th>\n",
       "      <td>000301971900007</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930660</th>\n",
       "      <td>000264005500007</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930661</th>\n",
       "      <td>000080163800019</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930662</th>\n",
       "      <td>000185745000010</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930663</th>\n",
       "      <td>A1982PH38400012</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930664</th>\n",
       "      <td>A1989AE93800008</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930665</th>\n",
       "      <td>000173977100041</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930666</th>\n",
       "      <td>000175801600018</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930667</th>\n",
       "      <td>000220747700018</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930668</th>\n",
       "      <td>000174377700007</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930669</th>\n",
       "      <td>000271586700013</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930670</th>\n",
       "      <td>000221871800012</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930671</th>\n",
       "      <td>A1954UF73200026</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930672</th>\n",
       "      <td>000176187500027</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930673</th>\n",
       "      <td>A1993LX59200002</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930674</th>\n",
       "      <td>000183262600008</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6930675</th>\n",
       "      <td>000321228000014</td>\n",
       "      <td>3</td>\n",
       "      <td>000321341000076</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>6930676 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        WoS_ID_reference section      WoS_ID_plos\n",
       "0        A1995QY75100004       0  000324515600133\n",
       "1        A1995QY75100004       3  000324515600133\n",
       "2        000222499800017       0  000324515600133\n",
       "3        000222499800017       3  000324515600133\n",
       "4        000263911400006       0  000324515600133\n",
       "5        000289279600018       0  000324515600133\n",
       "6        000289279600018       3  000324515600133\n",
       "7        000287717700023       0  000324515600133\n",
       "8        000257589600013       0  000324515600133\n",
       "9        000257589600013       3  000324515600133\n",
       "10       000246465200001       0  000324515600133\n",
       "11       000246465200001       3  000324515600133\n",
       "12       000238112100022       0  000324515600133\n",
       "13       000266227800006       0  000324515600133\n",
       "14       000266227800006       3  000324515600133\n",
       "15       000220879700004       0  000324515600133\n",
       "16       000177105200006       0  000324515600133\n",
       "17       000182540700009       0  000324515600133\n",
       "18       000227197300045       0  000324515600133\n",
       "19       000299607700013       0  000324515600133\n",
       "20       000290020400009       0  000324515600133\n",
       "21       000290020400009       3  000324515600133\n",
       "22       A1994NV95600001       0  000324515600133\n",
       "23       A1994NV95600001       3  000324515600133\n",
       "24       000256313500005       0  000324515600133\n",
       "25       000293136200014       0  000324515600133\n",
       "26       000174597100019       1  000324515600133\n",
       "27       000288797100011       2  000324515600133\n",
       "28       000179315000003       2  000324515600133\n",
       "29       000179315000003       3  000324515600133\n",
       "...                  ...     ...              ...\n",
       "6930646  000088538000012       0  000321341000076\n",
       "6930647  A1995RF14900014       0  000321341000076\n",
       "6930648  000167525500001       3  000321341000076\n",
       "6930649  A1992HZ04600007       0  000321341000076\n",
       "6930650  000268812400008       0  000321341000076\n",
       "6930651  000268812400008       3  000321341000076\n",
       "6930652  000285747900006       1  000321341000076\n",
       "6930653  000285747900006       3  000321341000076\n",
       "6930654  000090117600015       3  000321341000076\n",
       "6930655  000177854700008       3  000321341000076\n",
       "6930656  000247935300010       3  000321341000076\n",
       "6930657  A1982PW56700007       3  000321341000076\n",
       "6930658  A1980KX98200001       3  000321341000076\n",
       "6930659  000301971900007       3  000321341000076\n",
       "6930660  000264005500007       3  000321341000076\n",
       "6930661  000080163800019       3  000321341000076\n",
       "6930662  000185745000010       3  000321341000076\n",
       "6930663  A1982PH38400012       3  000321341000076\n",
       "6930664  A1989AE93800008       3  000321341000076\n",
       "6930665  000173977100041       3  000321341000076\n",
       "6930666  000175801600018       3  000321341000076\n",
       "6930667  000220747700018       3  000321341000076\n",
       "6930668  000174377700007       3  000321341000076\n",
       "6930669  000271586700013       3  000321341000076\n",
       "6930670  000221871800012       3  000321341000076\n",
       "6930671  A1954UF73200026       3  000321341000076\n",
       "6930672  000176187500027       3  000321341000076\n",
       "6930673  A1993LX59200002       3  000321341000076\n",
       "6930674  000183262600008       3  000321341000076\n",
       "6930675  000321228000014       3  000321341000076\n",
       "\n",
       "[6930676 rows x 3 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "testing_df3 = pd.read_csv('../data/df_plos_WoS_id_refs_sects_for_public.csv',sep=';',dtype='str', index_col='Unnamed: 0')\n",
    "print (\"done loading pickles\", testing_df3.shape)\n",
    "testing_df3\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Unnamed: 0          object\n",
       "WoS_ID_reference    object\n",
       "section             object\n",
       "WoS_ID_plos         object\n",
       "dtype: object"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "testing_df3.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}