99_process_CORE_sample.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "42fd8f4b",
   "metadata": {},
   "source": [
    "One-off code to turn the CORE sample (a bunch of nested folders) into a JSON "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f586904b",
   "metadata": {},
   "outputs": [],
   "source": [
    "#https://core.ac.uk/exports/resync_datadump_sample220218.zip\n",
    "#!unzip 'resync_datadump_sample220218.zip' -d \"datadump/\"\n",
    "#!mkdir jsons"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "b147b84b",
   "metadata": {},
   "outputs": [
    {
     "ename": "SyntaxError",
     "evalue": "invalid syntax (<ipython-input-11-6fcda8dd4500>, line 1)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;36m  File \u001b[0;32m\"<ipython-input-11-6fcda8dd4500>\"\u001b[0;36m, line \u001b[0;32m1\u001b[0m\n\u001b[0;31m    tar -xf downloads/140.tar.xz\u001b[0m\n\u001b[0m                    ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m invalid syntax\n"
     ]
    }
   ],
   "source": [
    "tar -x\"downloads/140.tar.xz\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "7eafcae4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/bin/sh: -c: line 0: syntax error near unexpected token `('\n",
      "/bin/sh: -c: line 0: `for i in `ls downloads/{100..300}.tar.xz`: print(i)'\n"
     ]
    }
   ],
   "source": [
    "for i in `ls {100..300}.tar.xz`\n",
    "do\n",
    "\ti\n",
    "done\n",
    "    \n",
    "for VARIABLE in 1 2 3 4 5 .. N\n",
    "do\n",
    "\tcommand2\n",
    "\tcommandN\n",
    "done"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "e1791bc9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/bin/sh: -c: line 1: syntax error: unexpected end of file\n",
      "/bin/sh: -c: line 0: syntax error near unexpected token `do'\n",
      "/bin/sh: -c: line 0: `do'\n",
      "ls: cannot access VARIABLE: No such file or directory\n",
      "/bin/sh: -c: line 0: syntax error near unexpected token `done'\n",
      "/bin/sh: -c: line 0: `done'\n"
     ]
    }
   ],
   "source": [
    "!for VARIABLE in \n",
    "!do\n",
    "!\tls VARIABLE\n",
    "!done"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "2637fe80",
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "43aac98f",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "935944it [00:13, 68321.36it/s]\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import json\n",
    "\n",
    "locs = []\n",
    "#150 is pubmed apparently\n",
    "\n",
    "\n",
    "k =0\n",
    "for root, dirs, files in tqdm(os.walk(\"downloads/pubtest\", topdown=False)):\n",
    "#     k+= 1\n",
    "#     if k > 500000:\n",
    "#         break\n",
    "    for name in files:    \n",
    "        fname = os.path.join(root, name)\n",
    "        if (fname.endswith(\".json\")):\n",
    "            locs.append(fname)\n",
    "\n",
    "\n",
    "#locs[0]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "ac63ac67",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2305355"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(locs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "b87b4cd5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[0m\u001b[01;34m00\u001b[0m/  \u001b[01;34m13\u001b[0m/  \u001b[01;34m22\u001b[0m/  \u001b[01;34m34\u001b[0m/  \u001b[01;34m45\u001b[0m/  \u001b[01;34m56\u001b[0m/  \u001b[01;34m67\u001b[0m/  \u001b[01;34m76\u001b[0m/  \u001b[01;34m86\u001b[0m/  \u001b[01;34m98\u001b[0m/  \u001b[01;34ma9\u001b[0m/  \u001b[01;34mb9\u001b[0m/  \u001b[01;34mc9\u001b[0m/  \u001b[01;34md9\u001b[0m/  \u001b[01;34me9\u001b[0m/  \u001b[01;34mfc\u001b[0m/\n",
      "\u001b[01;34m01\u001b[0m/  \u001b[01;34m14\u001b[0m/  \u001b[01;34m23\u001b[0m/  \u001b[01;34m35\u001b[0m/  \u001b[01;34m46\u001b[0m/  \u001b[01;34m57\u001b[0m/  \u001b[01;34m68\u001b[0m/  \u001b[01;34m77\u001b[0m/  \u001b[01;34m87\u001b[0m/  \u001b[01;34m99\u001b[0m/  \u001b[01;34maa\u001b[0m/  \u001b[01;34mbb\u001b[0m/  \u001b[01;34mca\u001b[0m/  \u001b[01;34mda\u001b[0m/  \u001b[01;34mea\u001b[0m/  \u001b[01;34mfd\u001b[0m/\n",
      "\u001b[01;34m02\u001b[0m/  \u001b[01;34m15\u001b[0m/  \u001b[01;34m24\u001b[0m/  \u001b[01;34m36\u001b[0m/  \u001b[01;34m47\u001b[0m/  \u001b[01;34m59\u001b[0m/  \u001b[01;34m69\u001b[0m/  \u001b[01;34m78\u001b[0m/  \u001b[01;34m88\u001b[0m/  \u001b[01;34m9a\u001b[0m/  \u001b[01;34mab\u001b[0m/  \u001b[01;34mbc\u001b[0m/  \u001b[01;34mcb\u001b[0m/  \u001b[01;34mdb\u001b[0m/  \u001b[01;34meb\u001b[0m/  \u001b[01;34mfe\u001b[0m/\n",
      "\u001b[01;34m04\u001b[0m/  \u001b[01;34m16\u001b[0m/  \u001b[01;34m25\u001b[0m/  \u001b[01;34m37\u001b[0m/  \u001b[01;34m49\u001b[0m/  \u001b[01;34m5a\u001b[0m/  \u001b[01;34m6a\u001b[0m/  \u001b[01;34m79\u001b[0m/  \u001b[01;34m8b\u001b[0m/  \u001b[01;34m9b\u001b[0m/  \u001b[01;34mac\u001b[0m/  \u001b[01;34mbd\u001b[0m/  \u001b[01;34mcc\u001b[0m/  \u001b[01;34mdc\u001b[0m/  \u001b[01;34mec\u001b[0m/  \u001b[01;34mff\u001b[0m/\n",
      "\u001b[01;34m05\u001b[0m/  \u001b[01;34m17\u001b[0m/  \u001b[01;34m26\u001b[0m/  \u001b[01;34m38\u001b[0m/  \u001b[01;34m4a\u001b[0m/  \u001b[01;34m5b\u001b[0m/  \u001b[01;34m6b\u001b[0m/  \u001b[01;34m7a\u001b[0m/  \u001b[01;34m8c\u001b[0m/  \u001b[01;34m9c\u001b[0m/  \u001b[01;34mad\u001b[0m/  \u001b[01;34mbe\u001b[0m/  \u001b[01;34mcd\u001b[0m/  \u001b[01;34mdd\u001b[0m/  \u001b[01;34med\u001b[0m/\n",
      "\u001b[01;34m06\u001b[0m/  \u001b[01;34m18\u001b[0m/  \u001b[01;34m28\u001b[0m/  \u001b[01;34m3a\u001b[0m/  \u001b[01;34m4b\u001b[0m/  \u001b[01;34m5c\u001b[0m/  \u001b[01;34m6c\u001b[0m/  \u001b[01;34m7b\u001b[0m/  \u001b[01;34m8d\u001b[0m/  \u001b[01;34m9e\u001b[0m/  \u001b[01;34mae\u001b[0m/  \u001b[01;34mbf\u001b[0m/  \u001b[01;34mce\u001b[0m/  \u001b[01;34mde\u001b[0m/  \u001b[01;34mee\u001b[0m/\n",
      "\u001b[01;34m08\u001b[0m/  \u001b[01;34m19\u001b[0m/  \u001b[01;34m29\u001b[0m/  \u001b[01;34m3b\u001b[0m/  \u001b[01;34m4c\u001b[0m/  \u001b[01;34m5d\u001b[0m/  \u001b[01;34m6d\u001b[0m/  \u001b[01;34m7c\u001b[0m/  \u001b[01;34m8e\u001b[0m/  \u001b[01;34m9f\u001b[0m/  \u001b[01;34maf\u001b[0m/  \u001b[01;34mc0\u001b[0m/  \u001b[01;34mcf\u001b[0m/  \u001b[01;34mdf\u001b[0m/  \u001b[01;34mef\u001b[0m/\n",
      "\u001b[01;34m09\u001b[0m/  \u001b[01;34m1a\u001b[0m/  \u001b[01;34m2a\u001b[0m/  \u001b[01;34m3c\u001b[0m/  \u001b[01;34m4e\u001b[0m/  \u001b[01;34m5e\u001b[0m/  \u001b[01;34m6e\u001b[0m/  \u001b[01;34m7d\u001b[0m/  \u001b[01;34m90\u001b[0m/  \u001b[01;34ma1\u001b[0m/  \u001b[01;34mb0\u001b[0m/  \u001b[01;34mc1\u001b[0m/  \u001b[01;34md0\u001b[0m/  \u001b[01;34me0\u001b[0m/  \u001b[01;34mf0\u001b[0m/\n",
      "\u001b[01;34m0a\u001b[0m/  \u001b[01;34m1b\u001b[0m/  \u001b[01;34m2b\u001b[0m/  \u001b[01;34m3d\u001b[0m/  \u001b[01;34m4f\u001b[0m/  \u001b[01;34m5f\u001b[0m/  \u001b[01;34m6f\u001b[0m/  \u001b[01;34m7e\u001b[0m/  \u001b[01;34m91\u001b[0m/  \u001b[01;34ma2\u001b[0m/  \u001b[01;34mb2\u001b[0m/  \u001b[01;34mc2\u001b[0m/  \u001b[01;34md1\u001b[0m/  \u001b[01;34me1\u001b[0m/  \u001b[01;34mf2\u001b[0m/\n",
      "\u001b[01;34m0b\u001b[0m/  \u001b[01;34m1c\u001b[0m/  \u001b[01;34m2c\u001b[0m/  \u001b[01;34m3e\u001b[0m/  \u001b[01;34m50\u001b[0m/  \u001b[01;34m60\u001b[0m/  \u001b[01;34m70\u001b[0m/  \u001b[01;34m7f\u001b[0m/  \u001b[01;34m92\u001b[0m/  \u001b[01;34ma3\u001b[0m/  \u001b[01;34mb3\u001b[0m/  \u001b[01;34mc3\u001b[0m/  \u001b[01;34md2\u001b[0m/  \u001b[01;34me2\u001b[0m/  \u001b[01;34mf3\u001b[0m/\n",
      "\u001b[01;34m0c\u001b[0m/  \u001b[01;34m1d\u001b[0m/  \u001b[01;34m2f\u001b[0m/  \u001b[01;34m3f\u001b[0m/  \u001b[01;34m51\u001b[0m/  \u001b[01;34m62\u001b[0m/  \u001b[01;34m71\u001b[0m/  \u001b[01;34m80\u001b[0m/  \u001b[01;34m93\u001b[0m/  \u001b[01;34ma4\u001b[0m/  \u001b[01;34mb4\u001b[0m/  \u001b[01;34mc4\u001b[0m/  \u001b[01;34md3\u001b[0m/  \u001b[01;34me3\u001b[0m/  \u001b[01;34mf4\u001b[0m/\n",
      "\u001b[01;34m0e\u001b[0m/  \u001b[01;34m1e\u001b[0m/  \u001b[01;34m30\u001b[0m/  \u001b[01;34m40\u001b[0m/  \u001b[01;34m52\u001b[0m/  \u001b[01;34m63\u001b[0m/  \u001b[01;34m72\u001b[0m/  \u001b[01;34m82\u001b[0m/  \u001b[01;34m94\u001b[0m/  \u001b[01;34ma5\u001b[0m/  \u001b[01;34mb5\u001b[0m/  \u001b[01;34mc5\u001b[0m/  \u001b[01;34md4\u001b[0m/  \u001b[01;34me4\u001b[0m/  \u001b[01;34mf5\u001b[0m/\n",
      "\u001b[01;34m0f\u001b[0m/  \u001b[01;34m1f\u001b[0m/  \u001b[01;34m31\u001b[0m/  \u001b[01;34m41\u001b[0m/  \u001b[01;34m53\u001b[0m/  \u001b[01;34m64\u001b[0m/  \u001b[01;34m73\u001b[0m/  \u001b[01;34m83\u001b[0m/  \u001b[01;34m95\u001b[0m/  \u001b[01;34ma6\u001b[0m/  \u001b[01;34mb6\u001b[0m/  \u001b[01;34mc6\u001b[0m/  \u001b[01;34md5\u001b[0m/  \u001b[01;34me5\u001b[0m/  \u001b[01;34mf7\u001b[0m/\n",
      "\u001b[01;34m10\u001b[0m/  \u001b[01;34m20\u001b[0m/  \u001b[01;34m32\u001b[0m/  \u001b[01;34m43\u001b[0m/  \u001b[01;34m54\u001b[0m/  \u001b[01;34m65\u001b[0m/  \u001b[01;34m74\u001b[0m/  \u001b[01;34m84\u001b[0m/  \u001b[01;34m96\u001b[0m/  \u001b[01;34ma7\u001b[0m/  \u001b[01;34mb7\u001b[0m/  \u001b[01;34mc7\u001b[0m/  \u001b[01;34md6\u001b[0m/  \u001b[01;34me7\u001b[0m/  \u001b[01;34mfa\u001b[0m/\n",
      "\u001b[01;34m12\u001b[0m/  \u001b[01;34m21\u001b[0m/  \u001b[01;34m33\u001b[0m/  \u001b[01;34m44\u001b[0m/  \u001b[01;34m55\u001b[0m/  \u001b[01;34m66\u001b[0m/  \u001b[01;34m75\u001b[0m/  \u001b[01;34m85\u001b[0m/  \u001b[01;34m97\u001b[0m/  \u001b[01;34ma8\u001b[0m/  \u001b[01;34mb8\u001b[0m/  \u001b[01;34mc8\u001b[0m/  \u001b[01;34md7\u001b[0m/  \u001b[01;34me8\u001b[0m/  \u001b[01;34mfb\u001b[0m/\n"
     ]
    }
   ],
   "source": [
    "#ls downloads/140"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "e4375b63",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2305355/2305355 [06:14<00:00, 6155.30it/s] \n"
     ]
    }
   ],
   "source": [
    "#d\n",
    "jsons = []\n",
    "\n",
    "for loc in tqdm(locs):\n",
    "    f = open(loc, \"r\")\n",
    "    data = json.load(f)\n",
    "    jsons.append(data)\n",
    "d = {\"data\": jsons}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "17734f00",
   "metadata": {},
   "outputs": [],
   "source": [
    "pubmed = [j for j in d[\"data\"] if \"pubmed\" in j[\"oai\"] and j[\"abstract\"]]\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "82875197",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.json_normalize(pubmed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "141aee09",
   "metadata": {},
   "outputs": [],
   "source": [
    "df2 = df\n",
    "df2 = df2.drop(columns=[\"abstract\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "8c5fd344",
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "\n",
    "import json\n",
    "import re\n",
    "\n",
    "\n",
    "alphabets= \"([A-Za-z])\"\n",
    "prefixes = \"(Mr|St|Mrs|Ms|Dr)[.]\"\n",
    "suffixes = \"(Inc|Ltd|Jr|Sr|Co)\"\n",
    "starters = \"(Mr|Mrs|Ms|Dr|He\\s|She\\s|It\\s|They\\s|Their\\s|Our\\s|We\\s|But\\s|However\\s|That\\s|This\\s|Wherever)\"\n",
    "acronyms = \"([A-Z][.][A-Z][.](?:[A-Z][.])?)\"\n",
    "websites = \"[.](com|net|org|io|gov)\"\n",
    "\n",
    "def split_into_sentences(text):\n",
    "    text = \" \" + text + \"  \"\n",
    "    text = text.replace(\"\\n\",\" \")\n",
    "    text = re.sub(prefixes,\"\\\\1<prd>\",text)\n",
    "    text = re.sub(websites,\"<prd>\\\\1\",text)\n",
    "    if \"Ph.D\" in text: text = text.replace(\"Ph.D.\",\"Ph<prd>D<prd>\")\n",
    "    text = re.sub(\"\\s\" + alphabets + \"[.] \",\" \\\\1<prd> \",text)\n",
    "    text = re.sub(acronyms+\" \"+starters,\"\\\\1<stop> \\\\2\",text)\n",
    "    text = re.sub(alphabets + \"[.]\" + alphabets + \"[.]\" + alphabets + \"[.]\",\"\\\\1<prd>\\\\2<prd>\\\\3<prd>\",text)\n",
    "    text = re.sub(alphabets + \"[.]\" + alphabets + \"[.]\",\"\\\\1<prd>\\\\2<prd>\",text)\n",
    "    text = re.sub(\" \"+suffixes+\"[.] \"+starters,\" \\\\1<stop> \\\\2\",text)\n",
    "    text = re.sub(\" \"+suffixes+\"[.]\",\" \\\\1<prd>\",text)\n",
    "    text = re.sub(\" \" + alphabets + \"[.]\",\" \\\\1<prd>\",text)\n",
    "    if \"”\" in text: text = text.replace(\".”\",\"”.\")\n",
    "    if \"\\\"\" in text: text = text.replace(\".\\\"\",\"\\\".\")\n",
    "    if \"!\" in text: text = text.replace(\"!\\\"\",\"\\\"!\")\n",
    "    if \"?\" in text: text = text.replace(\"?\\\"\",\"\\\"?\")\n",
    "    text = text.replace(\".\",\".<stop>\")\n",
    "    text = text.replace(\"?\",\"?<stop>\")\n",
    "    text = text.replace(\"!\",\"!<stop>\")\n",
    "    text = text.replace(\"<prd>\",\".\")\n",
    "    sentences = text.split(\"<stop>\")\n",
    "    sentences = sentences[:-1]\n",
    "    sentences = [s.strip() for s in sentences]\n",
    "    return sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "29899ee4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting pydash\n",
      "  Downloading pydash-5.1.0-py3-none-any.whl (84 kB)\n",
      "     |████████████████████████████████| 84 kB 507 kB/s             \n",
      "\u001b[?25hInstalling collected packages: pydash\n",
      "Successfully installed pydash-5.1.0\n"
     ]
    }
   ],
   "source": [
    "!pip install pydash\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "9da61bfc",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pydash import get"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "01436f1a",
   "metadata": {},
   "outputs": [],
   "source": [
    "df2[\"first_sent\"] = df[\"abstract\"].apply(lambda a: get(split_into_sentences(a), 0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "d36179d9",
   "metadata": {},
   "outputs": [],
   "source": [
    "df2.to_csv(\"downloads/core_first_sent.csv\", index = False, header=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "5a87a875",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>doi</th>\n",
       "      <th>coreId</th>\n",
       "      <th>oai</th>\n",
       "      <th>identifiers</th>\n",
       "      <th>title</th>\n",
       "      <th>authors</th>\n",
       "      <th>contributors</th>\n",
       "      <th>datePublished</th>\n",
       "      <th>downloadUrl</th>\n",
       "      <th>fullTextIdentifier</th>\n",
       "      <th>...</th>\n",
       "      <th>urls</th>\n",
       "      <th>issn</th>\n",
       "      <th>enrichments.references</th>\n",
       "      <th>enrichments.citationCount</th>\n",
       "      <th>enrichments.documentType.type</th>\n",
       "      <th>enrichments.documentType.confidence</th>\n",
       "      <th>language.code</th>\n",
       "      <th>language.name</th>\n",
       "      <th>language.id</th>\n",
       "      <th>first_sent</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>None</td>\n",
       "      <td>8361774</td>\n",
       "      <td>oai:pubmedcentral.nih.gov:2797167</td>\n",
       "      <td>[]</td>\n",
       "      <td>αvβ3 Integrin Limits the Contribution of Neuro...</td>\n",
       "      <td>[Robinson, Stephen D., Reynolds, Louise E., Ko...</td>\n",
       "      <td>[]</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>http://www.pubmedcentral.nih.gov/articlerender...</td>\n",
       "      <td>...</td>\n",
       "      <td>[http://www.pubmedcentral.nih.gov/articlerende...</td>\n",
       "      <td>None</td>\n",
       "      <td>[]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Both vascular endothelial growth factor recept...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10.1136/oem.20.4.277</td>\n",
       "      <td>3602553</td>\n",
       "      <td>oai:pubmedcentral.nih.gov:1038379</td>\n",
       "      <td>[]</td>\n",
       "      <td>Energy Expenditure Ranges and Muscular Work Gr...</td>\n",
       "      <td>[Brown, J. R., Crowden, G. P.]</td>\n",
       "      <td>[]</td>\n",
       "      <td>1963-10</td>\n",
       "      <td>None</td>\n",
       "      <td>http://www.pubmedcentral.nih.gov/articlerender...</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>1351-0711</td>\n",
       "      <td>[]</td>\n",
       "      <td>22.0</td>\n",
       "      <td>None</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>This paper is based on the findings of a field...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>10.1128/mcb.11.4.2282</td>\n",
       "      <td>3428118</td>\n",
       "      <td>oai:pubmedcentral.nih.gov:359930</td>\n",
       "      <td>[]</td>\n",
       "      <td>Characterization of a telomere-binding protein...</td>\n",
       "      <td>[Coren, J S, Epstein, E M, Vogt, V M]</td>\n",
       "      <td>[]</td>\n",
       "      <td>1991-04</td>\n",
       "      <td>None</td>\n",
       "      <td>http://www.pubmedcentral.nih.gov/articlerender...</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>0270-7306</td>\n",
       "      <td>[]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>We have partially purified a nuclear protein (...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>None</td>\n",
       "      <td>8208292</td>\n",
       "      <td>oai:pubmedcentral.nih.gov:2572022</td>\n",
       "      <td>[]</td>\n",
       "      <td>A Fortuitous Syncope. The pitfalls of Integrat...</td>\n",
       "      <td>[Salukhe, Tushar V, Wright, Ian, Wright, Matth...</td>\n",
       "      <td>[]</td>\n",
       "      <td>None</td>\n",
       "      <td>https://core.ac.uk/download/pdf/8208292.pdf</td>\n",
       "      <td>http://www.pubmedcentral.nih.gov/articlerender...</td>\n",
       "      <td>...</td>\n",
       "      <td>[http://www.pubmedcentral.nih.gov/articlerende...</td>\n",
       "      <td>None</td>\n",
       "      <td>[{'date': '1999', 'raw': 'Babuty D, Fauchier L...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>NaN</td>\n",
       "      <td>en</td>\n",
       "      <td>English</td>\n",
       "      <td>9.0</td>\n",
       "      <td>Myopotential oversensing in implantable defibr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>None</td>\n",
       "      <td>8364895</td>\n",
       "      <td>oai:pubmedcentral.nih.gov:2801609</td>\n",
       "      <td>[]</td>\n",
       "      <td>Control of Alzheimer's Amyloid Beta Toxicity b...</td>\n",
       "      <td>[Sanokawa-Akakura, Reiko, Cao, Weihuan, Allan,...</td>\n",
       "      <td>[]</td>\n",
       "      <td>None</td>\n",
       "      <td>https://core.ac.uk/download/pdf/8364895.pdf</td>\n",
       "      <td>http://www.pubmedcentral.nih.gov/articlerender...</td>\n",
       "      <td>...</td>\n",
       "      <td>[http://www.pubmedcentral.nih.gov/articlerende...</td>\n",
       "      <td>None</td>\n",
       "      <td>[{'date': '1986', 'raw': 'Tai PK, Maeda Y, Nak...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>NaN</td>\n",
       "      <td>en</td>\n",
       "      <td>English</td>\n",
       "      <td>9.0</td>\n",
       "      <td>FK506 binding proteins (FKBPs), also called im...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1144178</th>\n",
       "      <td>10.1128/jb.170.3.1403-1404.1988</td>\n",
       "      <td>3295288</td>\n",
       "      <td>oai:pubmedcentral.nih.gov:210926</td>\n",
       "      <td>[]</td>\n",
       "      <td>Properties of spores of Bacillus subtilis stra...</td>\n",
       "      <td>[Hackett, R H, Setlow, P]</td>\n",
       "      <td>[]</td>\n",
       "      <td>1988-03</td>\n",
       "      <td>None</td>\n",
       "      <td>http://www.pubmedcentral.nih.gov/articlerender...</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>0021-9193</td>\n",
       "      <td>[]</td>\n",
       "      <td>31.0</td>\n",
       "      <td>None</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Bacillus subtilis strains containing a deletio...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1144179</th>\n",
       "      <td>None</td>\n",
       "      <td>8061207</td>\n",
       "      <td>oai:pubmedcentral.nih.gov:2393453</td>\n",
       "      <td>[]</td>\n",
       "      <td>Health, safe water and sanitation: a cross-sec...</td>\n",
       "      <td>[Wibowo, D., Tisdell, C.]</td>\n",
       "      <td>[]</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>http://www.pubmedcentral.nih.gov/articlerender...</td>\n",
       "      <td>...</td>\n",
       "      <td>[http://www.pubmedcentral.nih.gov/articlerende...</td>\n",
       "      <td>None</td>\n",
       "      <td>[]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>The study describes the development of health ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1144180</th>\n",
       "      <td>None</td>\n",
       "      <td>3892463</td>\n",
       "      <td>oai:pubmedcentral.nih.gov:1388572</td>\n",
       "      <td>[]</td>\n",
       "      <td>Reduction of Nitrated Diphenylamine Derivative...</td>\n",
       "      <td>[Drzyzga, O., Schmidt, A., Blotevogel, K.]</td>\n",
       "      <td>[]</td>\n",
       "      <td>1995-09</td>\n",
       "      <td>None</td>\n",
       "      <td>http://www.pubmedcentral.nih.gov/articlerender...</td>\n",
       "      <td>...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>[]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2-Nitrodiphenylamine, 4-nitrodiphenylamine, an...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1144181</th>\n",
       "      <td>None</td>\n",
       "      <td>8585567</td>\n",
       "      <td>oai:pubmedcentral.nih.gov:3123099</td>\n",
       "      <td>[]</td>\n",
       "      <td>Active Site-labeled Prothrombin Inhibits Proth...</td>\n",
       "      <td>[Kroh, Heather K., Panizzi, Peter, Tchaikovski...</td>\n",
       "      <td>[]</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>http://www.pubmedcentral.nih.gov/articlerender...</td>\n",
       "      <td>...</td>\n",
       "      <td>[http://www.pubmedcentral.nih.gov/articlerende...</td>\n",
       "      <td>None</td>\n",
       "      <td>[]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Mouse and human prothrombin (ProT) active site...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1144182</th>\n",
       "      <td>None</td>\n",
       "      <td>8304805</td>\n",
       "      <td>oai:pubmedcentral.nih.gov:2708412</td>\n",
       "      <td>[]</td>\n",
       "      <td>Major Species-Specific Antibody Epitopes of th...</td>\n",
       "      <td>[Luo, Tian, Zhang, Xiaofeng, McBride, Jere W.]</td>\n",
       "      <td>[]</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>http://www.pubmedcentral.nih.gov/articlerender...</td>\n",
       "      <td>...</td>\n",
       "      <td>[http://www.pubmedcentral.nih.gov/articlerende...</td>\n",
       "      <td>None</td>\n",
       "      <td>[]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Ehrlichia chaffeensis and E. canis have a smal...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1144183 rows × 30 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     doi   coreId  \\\n",
       "0                                   None  8361774   \n",
       "1                   10.1136/oem.20.4.277  3602553   \n",
       "2                  10.1128/mcb.11.4.2282  3428118   \n",
       "3                                   None  8208292   \n",
       "4                                   None  8364895   \n",
       "...                                  ...      ...   \n",
       "1144178  10.1128/jb.170.3.1403-1404.1988  3295288   \n",
       "1144179                             None  8061207   \n",
       "1144180                             None  3892463   \n",
       "1144181                             None  8585567   \n",
       "1144182                             None  8304805   \n",
       "\n",
       "                                       oai identifiers  \\\n",
       "0        oai:pubmedcentral.nih.gov:2797167          []   \n",
       "1        oai:pubmedcentral.nih.gov:1038379          []   \n",
       "2         oai:pubmedcentral.nih.gov:359930          []   \n",
       "3        oai:pubmedcentral.nih.gov:2572022          []   \n",
       "4        oai:pubmedcentral.nih.gov:2801609          []   \n",
       "...                                    ...         ...   \n",
       "1144178   oai:pubmedcentral.nih.gov:210926          []   \n",
       "1144179  oai:pubmedcentral.nih.gov:2393453          []   \n",
       "1144180  oai:pubmedcentral.nih.gov:1388572          []   \n",
       "1144181  oai:pubmedcentral.nih.gov:3123099          []   \n",
       "1144182  oai:pubmedcentral.nih.gov:2708412          []   \n",
       "\n",
       "                                                     title  \\\n",
       "0        αvβ3 Integrin Limits the Contribution of Neuro...   \n",
       "1        Energy Expenditure Ranges and Muscular Work Gr...   \n",
       "2        Characterization of a telomere-binding protein...   \n",
       "3        A Fortuitous Syncope. The pitfalls of Integrat...   \n",
       "4        Control of Alzheimer's Amyloid Beta Toxicity b...   \n",
       "...                                                    ...   \n",
       "1144178  Properties of spores of Bacillus subtilis stra...   \n",
       "1144179  Health, safe water and sanitation: a cross-sec...   \n",
       "1144180  Reduction of Nitrated Diphenylamine Derivative...   \n",
       "1144181  Active Site-labeled Prothrombin Inhibits Proth...   \n",
       "1144182  Major Species-Specific Antibody Epitopes of th...   \n",
       "\n",
       "                                                   authors contributors  \\\n",
       "0        [Robinson, Stephen D., Reynolds, Louise E., Ko...           []   \n",
       "1                           [Brown, J. R., Crowden, G. P.]           []   \n",
       "2                    [Coren, J S, Epstein, E M, Vogt, V M]           []   \n",
       "3        [Salukhe, Tushar V, Wright, Ian, Wright, Matth...           []   \n",
       "4        [Sanokawa-Akakura, Reiko, Cao, Weihuan, Allan,...           []   \n",
       "...                                                    ...          ...   \n",
       "1144178                          [Hackett, R H, Setlow, P]           []   \n",
       "1144179                          [Wibowo, D., Tisdell, C.]           []   \n",
       "1144180         [Drzyzga, O., Schmidt, A., Blotevogel, K.]           []   \n",
       "1144181  [Kroh, Heather K., Panizzi, Peter, Tchaikovski...           []   \n",
       "1144182     [Luo, Tian, Zhang, Xiaofeng, McBride, Jere W.]           []   \n",
       "\n",
       "        datePublished                                  downloadUrl  \\\n",
       "0                None                                         None   \n",
       "1             1963-10                                         None   \n",
       "2             1991-04                                         None   \n",
       "3                None  https://core.ac.uk/download/pdf/8208292.pdf   \n",
       "4                None  https://core.ac.uk/download/pdf/8364895.pdf   \n",
       "...               ...                                          ...   \n",
       "1144178       1988-03                                         None   \n",
       "1144179          None                                         None   \n",
       "1144180       1995-09                                         None   \n",
       "1144181          None                                         None   \n",
       "1144182          None                                         None   \n",
       "\n",
       "                                        fullTextIdentifier  ...  \\\n",
       "0        http://www.pubmedcentral.nih.gov/articlerender...  ...   \n",
       "1        http://www.pubmedcentral.nih.gov/articlerender...  ...   \n",
       "2        http://www.pubmedcentral.nih.gov/articlerender...  ...   \n",
       "3        http://www.pubmedcentral.nih.gov/articlerender...  ...   \n",
       "4        http://www.pubmedcentral.nih.gov/articlerender...  ...   \n",
       "...                                                    ...  ...   \n",
       "1144178  http://www.pubmedcentral.nih.gov/articlerender...  ...   \n",
       "1144179  http://www.pubmedcentral.nih.gov/articlerender...  ...   \n",
       "1144180  http://www.pubmedcentral.nih.gov/articlerender...  ...   \n",
       "1144181  http://www.pubmedcentral.nih.gov/articlerender...  ...   \n",
       "1144182  http://www.pubmedcentral.nih.gov/articlerender...  ...   \n",
       "\n",
       "                                                      urls       issn  \\\n",
       "0        [http://www.pubmedcentral.nih.gov/articlerende...       None   \n",
       "1                                                     None  1351-0711   \n",
       "2                                                     None  0270-7306   \n",
       "3        [http://www.pubmedcentral.nih.gov/articlerende...       None   \n",
       "4        [http://www.pubmedcentral.nih.gov/articlerende...       None   \n",
       "...                                                    ...        ...   \n",
       "1144178                                               None  0021-9193   \n",
       "1144179  [http://www.pubmedcentral.nih.gov/articlerende...       None   \n",
       "1144180                                               None       None   \n",
       "1144181  [http://www.pubmedcentral.nih.gov/articlerende...       None   \n",
       "1144182  [http://www.pubmedcentral.nih.gov/articlerende...       None   \n",
       "\n",
       "                                    enrichments.references  \\\n",
       "0                                                       []   \n",
       "1                                                       []   \n",
       "2                                                       []   \n",
       "3        [{'date': '1999', 'raw': 'Babuty D, Fauchier L...   \n",
       "4        [{'date': '1986', 'raw': 'Tai PK, Maeda Y, Nak...   \n",
       "...                                                    ...   \n",
       "1144178                                                 []   \n",
       "1144179                                                 []   \n",
       "1144180                                                 []   \n",
       "1144181                                                 []   \n",
       "1144182                                                 []   \n",
       "\n",
       "        enrichments.citationCount  enrichments.documentType.type  \\\n",
       "0                             NaN                           None   \n",
       "1                            22.0                           None   \n",
       "2                             NaN                           None   \n",
       "3                             NaN                           None   \n",
       "4                             NaN                           None   \n",
       "...                           ...                            ...   \n",
       "1144178                      31.0                           None   \n",
       "1144179                       NaN                           None   \n",
       "1144180                       NaN                           None   \n",
       "1144181                       NaN                           None   \n",
       "1144182                       NaN                           None   \n",
       "\n",
       "        enrichments.documentType.confidence  language.code language.name  \\\n",
       "0                                       NaN            NaN           NaN   \n",
       "1                                       NaN            NaN           NaN   \n",
       "2                                       NaN            NaN           NaN   \n",
       "3                                       NaN             en       English   \n",
       "4                                       NaN             en       English   \n",
       "...                                     ...            ...           ...   \n",
       "1144178                                 NaN            NaN           NaN   \n",
       "1144179                                 NaN            NaN           NaN   \n",
       "1144180                                 NaN            NaN           NaN   \n",
       "1144181                                 NaN            NaN           NaN   \n",
       "1144182                                 NaN            NaN           NaN   \n",
       "\n",
       "        language.id                                         first_sent  \n",
       "0               NaN  Both vascular endothelial growth factor recept...  \n",
       "1               NaN  This paper is based on the findings of a field...  \n",
       "2               NaN  We have partially purified a nuclear protein (...  \n",
       "3               9.0  Myopotential oversensing in implantable defibr...  \n",
       "4               9.0  FK506 binding proteins (FKBPs), also called im...  \n",
       "...             ...                                                ...  \n",
       "1144178         NaN  Bacillus subtilis strains containing a deletio...  \n",
       "1144179         NaN  The study describes the development of health ...  \n",
       "1144180         NaN  2-Nitrodiphenylamine, 4-nitrodiphenylamine, an...  \n",
       "1144181         NaN  Mouse and human prothrombin (ProT) active site...  \n",
       "1144182         NaN  Ehrlichia chaffeensis and E. canis have a smal...  \n",
       "\n",
       "[1144183 rows x 30 columns]"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "7cff0887",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv(\"downloads/core3.csv\", index = False, header=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bab6a79e",
   "metadata": {},
   "source": [
    "## Archived Code Snippets "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cf704fee",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('downloads/CORE_out.json') as f:\n",
    "    g = f.read()\n",
    "    data = json.loads(g)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3c1a78f5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(21841, 7595)"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data =d\n",
    "hits = [paper for paper in data['data'] if paper[\"abstract\"] and paper[\"fullText\"] and len(paper[\"abstract\"])> 15]\n",
    "len(data[\"data\"]), len(hits)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3616ae25",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "#from utils\n",
    "coll = [{\"abstract\": e[\"abstract\"], \"fullText\": e[\"fullText\"], \"doi\": e[\"doi\"]} for e in hits]\n",
    "coll[0]\n",
    "[c.update(\n",
    "    {\"keysentence_abstract\": find_sentence_in_abstract(c[\"abstract\"]), \n",
    "    \"sent_before_ABST\":find_sentence_in_abstract(c[\"abstract\"], -1),\n",
    "    \"sent_after_ABST\":find_sentence_in_abstract(c[\"abstract\"], -1),\n",
    "\n",
    "     \"keysentence_fulltext\": find_sentence_in_abstract(c[\"fullText\"]),\n",
    "     \"sent_before_FT\":find_sentence_in_abstract(c[\"fullText\"], -1),\n",
    "     \"sent_after_FT\": find_sentence_in_abstract(c[\"fullText\"], 1),\n",
    "     }) for c in coll]\n",
    "\n",
    "4\n",
    "\n",
    "[c.update({\"sentiment\": sentiment(c[\"phrase\"])}) for c in phrase_hits]\n",
    "[c.update({\"modality\": modality(c[\"phrase\"]) }) for c in phrase_hits]\n",
    "\n",
    "4\n",
    "only_both = [c for c in coll if c[\"keysentence_abstract\"] and c[\"keysentence_fulltext\"]]\n",
    "len(only_both)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "conda_python3",
   "language": "python",
   "name": "conda_python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}