integrate_train.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "27330ef3",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "4663255b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             total       used       free     shared    buffers     cached\n",
      "Mem:     261128012    3346036  257781976         80     173056    1698476\n",
      "-/+ buffers/cache:    1474504  259653508\n",
      "Swap:            0          0          0\n"
     ]
    }
   ],
   "source": [
    "#!find . -maxdepth 3 -type f -exec du -h {} + | sort --human-numeric-sort --reverse\n",
    "!free"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cc8cae4f",
   "metadata": {},
   "source": [
    "## INTEGRATE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "226e373d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('The northern corn rootworm', 'The northern corn rootworm')"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "blacklist=[\"abstract\", \"discussion\", \"background\", \"summary\", \"rationale\", \"introduction\", \"publisher\", \"introduction\", \"statement\", \"purpose\", \"objective\" ]\n",
    "blacklist += [t + \":\" for t in blacklist]\n",
    "def remove_special_words(text, blacklist=blacklist ):\n",
    "    words = text.split(\" \")\n",
    "    try:\n",
    "        first=words[0]\n",
    "    except:\n",
    "        return text\n",
    "    \n",
    "    first = first.strip().lower()\n",
    "#     if any(first in string for string in blacklist):\n",
    "#         return \" \".join(words[1:]).strip()\n",
    "    for w in reversed(blacklist): #prioritize \"Abstract:The\" replace over \"AbstractThe\"\n",
    "        if w in first:\n",
    "            return \" \".join([first.split(w)[1].capitalize()] + words[1:]).strip()\n",
    "    return text\n",
    "\n",
    "remove_special_words(\"ABSTRACT   The northern corn rootworm\"), remove_special_words(\"purpose:The northern corn rootworm\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "cec91497",
   "metadata": {},
   "outputs": [],
   "source": [
    "df2 = pd.read_csv(\"downloads/train_june20_all.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "99579d99",
   "metadata": {},
   "outputs": [],
   "source": [
    "#df2[df2[\"text\"].str.contains(\"Vietnam\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "bd7475e1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['acl_cambridge', 'predicts_acl_pm2500_vocabed',\n",
       "       'Oct1_clinical_studies_pm', 'oct3_labels', 'labels_oct7',\n",
       "       ' limitation|problem ... with ... l=be that', 'a ... drawback of',\n",
       "       'a {negative_adjectives} ...  {problem_nouns}', 'pm_OBJECTIVE',\n",
       "       'pm_METHODS', 'pm_RESULTS', 'pm_CONCLUSIONS', 'biology1k_050522',\n",
       "       'psych1k_100522', 'june152022_labels_envgeosoc',\n",
       "       'june152022_labels_matscichem'], dtype=object)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2[\"source\"].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "bc80ce9f",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "url = \"https://docs.google.com/spreadsheets/d/e/2PACX-1vSrGaIXKQZOe0960YVYXn3z1aUpUNfsbYzFoUOv9-wGf84oLJCYr_KA2XzxeG0rUi0ff5lWCtHxmgsg/pub?output=csv\"\n",
    "url1 = \"https://docs.google.com/spreadsheets/d/e/2PACX-1vSMOZaJsr2yRdX7SX5xLyvtd2fZXkj3DicRlE032sTbiBWinLtOGRX33eY6ewsm5jMv4xnsXMprHEqn/pub?output=csv\"\n",
    "#url2= \"https://docs.google.com/spreadsheets/d/e/2PACX-1vSXaZ3xwCNvUTs3DgqzCaE3W481KJQhC6FqbNQ6zU04AGCj37vqS5TEMidyh46NC6UlyCbSXtj52UOc/pub?output=csv\"\n",
    "#url3 = \"https://docs.google.com/spreadsheets/d/e/2PACX-1vSkLXuuISZkDh53ttC9AZ1VvDaMXfqoOFSUqYHyzDHStlqNsbi3fiQoe993TSKoay5UyFHsG_otlCGO/pub?output=csv\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "869158ad",
   "metadata": {},
   "outputs": [],
   "source": [
    "#source = \"june152022_labels_envgeosoc\"\n",
    "source = \"june152022_labels_matscichem\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "fe12aa8e",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(url)\n",
    "#df2 = pd.read_csv(url2)\n",
    "df0 = pd.read_csv(url1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "c557a59c",
   "metadata": {},
   "outputs": [],
   "source": [
    "df0[\"source\"] = \"june152022_labels_envgeosoc\"\n",
    "df[\"source\"] = \"june152022_labels_matscichem\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "id": "0ba7deeb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1569"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df.append(df0)\n",
    "len(df0[df0.label.notna()])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "f62426fd",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ec2-user/anaconda3/envs/python3/lib/python3.6/site-packages/ipykernel/__main__.py:6: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    }
   ],
   "source": [
    "len(df)\n",
    "df[\"labels\"] = df[\"label\"]\n",
    "# df[\"source\"] = source\n",
    "#df = df.drop(columns=[\"predicts\"])\n",
    "df = df.dropna(subset=[\"labels\"])\n",
    "df[\"labels\"] = df[\"labels\"].astype(int) \n",
    "df = df[(df[\"labels\"] == 1) | (df[\"labels\"] == 0)] #fix typos in labels eg. misstyped 0 as 9\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "id": "c14b957c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2319"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df.drop_duplicates(subset=[\"text\"]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "5760d809",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "44632"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "eab46056",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>doi</th>\n",
       "      <th>journalName</th>\n",
       "      <th>title</th>\n",
       "      <th>fields</th>\n",
       "      <th>source</th>\n",
       "      <th>labels</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>The difficulty with this task lies in the fact...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Computer Science</td>\n",
       "      <td>acl_cambridge</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>The problem with rich annotations is that they...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Computer Science</td>\n",
       "      <td>acl_cambridge</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>As a consequence , when adapting existing meth...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Computer Science</td>\n",
       "      <td>acl_cambridge</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>The second problem of traditional word alignme...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Computer Science</td>\n",
       "      <td>acl_cambridge</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>The main drawback of these systems is that the...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Computer Science</td>\n",
       "      <td>acl_cambridge</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44627</th>\n",
       "      <td>Approximately 50% of patients with schizophren...</td>\n",
       "      <td>10.1001/ARCHPSYC.60.1.82</td>\n",
       "      <td>Archives of general psychiatry</td>\n",
       "      <td>Clozapine treatment for suicidality in schizop...</td>\n",
       "      <td>Psychology</td>\n",
       "      <td>june152022_labels_matscichem</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44628</th>\n",
       "      <td>Isoconversion processing of thermoanalytical d...</td>\n",
       "      <td>10.1016/J.TCA.2008.07.008</td>\n",
       "      <td>Thermochimica Acta</td>\n",
       "      <td>On problems of isoconversion data processing f...</td>\n",
       "      <td>Materials Science</td>\n",
       "      <td>june152022_labels_matscichem</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44629</th>\n",
       "      <td>The lack of social attributes in online teachi...</td>\n",
       "      <td>10.1109/CSEI50228.2020.9142498</td>\n",
       "      <td>2020 IEEE 2nd International Conference on Comp...</td>\n",
       "      <td>The Presence Design of Online Teaching based o...</td>\n",
       "      <td>Psychology</td>\n",
       "      <td>june152022_labels_matscichem</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44630</th>\n",
       "      <td>Subsurface inspection of food and agricultural...</td>\n",
       "      <td>10.1016/J.FOODCONT.2016.12.012</td>\n",
       "      <td>Food Control</td>\n",
       "      <td>Subsurface inspection of food safety and quali...</td>\n",
       "      <td>Materials Science</td>\n",
       "      <td>june152022_labels_matscichem</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44631</th>\n",
       "      <td>Evaporation of a moving liquid droplet in atmo...</td>\n",
       "      <td>10.1016/J.POWTEC.2012.07.002</td>\n",
       "      <td>Powder Technology</td>\n",
       "      <td>Evaluation of Evaporation Models for Single Mo...</td>\n",
       "      <td>Chemistry</td>\n",
       "      <td>june152022_labels_matscichem</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>44632 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    text  \\\n",
       "0      The difficulty with this task lies in the fact...   \n",
       "1      The problem with rich annotations is that they...   \n",
       "2      As a consequence , when adapting existing meth...   \n",
       "3      The second problem of traditional word alignme...   \n",
       "4      The main drawback of these systems is that the...   \n",
       "...                                                  ...   \n",
       "44627  Approximately 50% of patients with schizophren...   \n",
       "44628  Isoconversion processing of thermoanalytical d...   \n",
       "44629  The lack of social attributes in online teachi...   \n",
       "44630  Subsurface inspection of food and agricultural...   \n",
       "44631  Evaporation of a moving liquid droplet in atmo...   \n",
       "\n",
       "                                  doi  \\\n",
       "0                                 NaN   \n",
       "1                                 NaN   \n",
       "2                                 NaN   \n",
       "3                                 NaN   \n",
       "4                                 NaN   \n",
       "...                               ...   \n",
       "44627        10.1001/ARCHPSYC.60.1.82   \n",
       "44628       10.1016/J.TCA.2008.07.008   \n",
       "44629  10.1109/CSEI50228.2020.9142498   \n",
       "44630  10.1016/J.FOODCONT.2016.12.012   \n",
       "44631    10.1016/J.POWTEC.2012.07.002   \n",
       "\n",
       "                                             journalName  \\\n",
       "0                                                    NaN   \n",
       "1                                                    NaN   \n",
       "2                                                    NaN   \n",
       "3                                                    NaN   \n",
       "4                                                    NaN   \n",
       "...                                                  ...   \n",
       "44627                     Archives of general psychiatry   \n",
       "44628                                 Thermochimica Acta   \n",
       "44629  2020 IEEE 2nd International Conference on Comp...   \n",
       "44630                                       Food Control   \n",
       "44631                                  Powder Technology   \n",
       "\n",
       "                                                   title             fields  \\\n",
       "0                                                    NaN   Computer Science   \n",
       "1                                                    NaN   Computer Science   \n",
       "2                                                    NaN   Computer Science   \n",
       "3                                                    NaN   Computer Science   \n",
       "4                                                    NaN   Computer Science   \n",
       "...                                                  ...                ...   \n",
       "44627  Clozapine treatment for suicidality in schizop...         Psychology   \n",
       "44628  On problems of isoconversion data processing f...  Materials Science   \n",
       "44629  The Presence Design of Online Teaching based o...         Psychology   \n",
       "44630  Subsurface inspection of food safety and quali...  Materials Science   \n",
       "44631  Evaluation of Evaporation Models for Single Mo...          Chemistry   \n",
       "\n",
       "                             source  labels  \n",
       "0                     acl_cambridge       1  \n",
       "1                     acl_cambridge       1  \n",
       "2                     acl_cambridge       1  \n",
       "3                     acl_cambridge       1  \n",
       "4                     acl_cambridge       1  \n",
       "...                             ...     ...  \n",
       "44627  june152022_labels_matscichem       1  \n",
       "44628  june152022_labels_matscichem       1  \n",
       "44629  june152022_labels_matscichem       1  \n",
       "44630  june152022_labels_matscichem       1  \n",
       "44631  june152022_labels_matscichem       0  \n",
       "\n",
       "[44632 rows x 7 columns]"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2.drop_duplicates(subset=[\"text\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "id": "66f1159e",
   "metadata": {},
   "outputs": [],
   "source": [
    "df4 = df2.append(df)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "id": "c7c295e7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "44687"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df4.drop_duplicates(subset=[\"text\"]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "f6c5062e",
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'DataFrame' object has no attribute 'source'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-84-e8be80cb0a56>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     17\u001b[0m }\n\u001b[1;32m     18\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 19\u001b[0;31m \u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"fields2\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msource\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmap\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mlambda\u001b[0m \u001b[0ms\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0ms2f\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     20\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"fields\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfields\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcombine_first\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'fields2'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     21\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/python3/lib/python3.6/site-packages/pandas/core/generic.py\u001b[0m in \u001b[0;36m__getattr__\u001b[0;34m(self, name)\u001b[0m\n\u001b[1;32m   5139\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_info_axis\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_can_hold_identifiers_and_holds_name\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   5140\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 5141\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mobject\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__getattribute__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   5142\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   5143\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__setattr__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mAttributeError\u001b[0m: 'DataFrame' object has no attribute 'source'"
     ]
    }
   ],
   "source": [
    "s2f = {\n",
    "    \"pm_METHODS\": \"Medicine\",\n",
    "    \"pm_RESULTS\": \"Medicine\",\n",
    "    \"pm_CONCLUSIONS\": \"Medicine\",\n",
    "    \"pm_OBJECTIVE\": \"Medicine\",\n",
    "    \"pm_OBJECTIVE\": \"Medicine\",\n",
    "    \"a {negative_adjectives} ...  {problem_nouns}\": \"Medicine\",\n",
    "    \"oct3_labels\": \"Medicine\",\n",
    "    \" limitation|problem ... with ... l=be that\": \"Medicine\",\n",
    "    \"labels_oct7\": \"Medicine\",\n",
    "    \"Oct1_clinical_studies_pm\": \"Medicine\",\n",
    "    \"predicts_acl_pm2500_vocabed\": \"Medicine\",\n",
    "    \"Oct1_clinical_studies_pm\": \"Medicine\",\n",
    "    \"acl_cambridge\": \"Computer Science\",\n",
    "    \"psych1k_100522\": \"Psychology\",\n",
    "    \"biology1k_050522\": \"Biology\",   \n",
    "}\n",
    "\n",
    "df[\"fields2\"] = df.source.map(lambda s: s2f.get(s)) \n",
    "df[\"fields\"] = df.fields.combine_first(df['fields2'])\n",
    "\n",
    "df[\"fields\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "dbb391c8",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "57198d68",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "3966c42d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "46951"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#df2 = pd.read_csv(\"downloads/43k_pm_bio.csv\")\n",
    "df3 = df2.append(df)\n",
    "len(df3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "f23dc97b",
   "metadata": {},
   "outputs": [],
   "source": [
    "df3 = df3.drop_duplicates(subset=[\"text\"]) #doi with nans not good\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "8c4e3f32",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "44687"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "61a368ea",
   "metadata": {},
   "outputs": [],
   "source": [
    "df3 =df3[[\"text\", \"doi\", \"journalName\", \"title\", \"fields\", \"source\", \"labels\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "378d0c54",
   "metadata": {},
   "outputs": [],
   "source": [
    "df3[\"text\"] = df3.text.map(remove_special_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "dc10c7f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "excluded = df3[df3.fields.isin([\"Chemistry\", \"Political Science\"])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "76964eac",
   "metadata": {},
   "outputs": [],
   "source": [
    "excluded.to_csv (\"downloads/chem_polsci_excluded_300_labels.csv\", index = False, header=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "d9910f72",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_final = df3[~df3.fields.isin([\"Chemistry\", \"Political Science\"])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "8b6813ae",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_final.to_csv(\"downloads/train_final_june23_all.csv\", index = False, header=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "ecd523df",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>doi</th>\n",
       "      <th>journalName</th>\n",
       "      <th>title</th>\n",
       "      <th>fields</th>\n",
       "      <th>source</th>\n",
       "      <th>labels</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>The difficulty with this task lies in the fact...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Computer Science</td>\n",
       "      <td>acl_cambridge</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>The problem with rich annotations is that they...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Computer Science</td>\n",
       "      <td>acl_cambridge</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>As a consequence , when adapting existing meth...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Computer Science</td>\n",
       "      <td>acl_cambridge</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>The second problem of traditional word alignme...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Computer Science</td>\n",
       "      <td>acl_cambridge</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>The main drawback of these systems is that the...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Computer Science</td>\n",
       "      <td>acl_cambridge</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1361</th>\n",
       "      <td>The area used for bioenergy crops (annual row ...</td>\n",
       "      <td>10.1186/S13705-015-0067-3</td>\n",
       "      <td>Energy, Sustainability and Society</td>\n",
       "      <td>Legume-based mixed intercropping systems may l...</td>\n",
       "      <td>Environmental Science</td>\n",
       "      <td>june152022_labels_envgeosoc</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1421</th>\n",
       "      <td>Improper disposal of used polyethylene terepht...</td>\n",
       "      <td>10.1007/s11367-012-0447-2</td>\n",
       "      <td>The International Journal of Life Cycle Assess...</td>\n",
       "      <td>Comparative life cycle assessment and social l...</td>\n",
       "      <td>Environmental Science</td>\n",
       "      <td>june152022_labels_envgeosoc</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1493</th>\n",
       "      <td>Airborne transmission of pathogenic aerosols v...</td>\n",
       "      <td>10.4209/AAQR.2016.09.0400</td>\n",
       "      <td>Aerosol and Air Quality Research</td>\n",
       "      <td>Fluorescent Bioaerosol Particles Resulting fro...</td>\n",
       "      <td>Environmental Science</td>\n",
       "      <td>june152022_labels_envgeosoc</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1513</th>\n",
       "      <td>Lead is one of the hazardous materials which i...</td>\n",
       "      <td>10.1186/2052-336X-12-103</td>\n",
       "      <td>Journal of Environmental Health Science and En...</td>\n",
       "      <td>Covalently modified magnetite nanoparticles wi...</td>\n",
       "      <td>Environmental Science</td>\n",
       "      <td>june152022_labels_envgeosoc</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1548</th>\n",
       "      <td>Controlling airborne contamination is of major...</td>\n",
       "      <td>10.1186/1471-2334-11-58</td>\n",
       "      <td>BMC Infectious Diseases</td>\n",
       "      <td>Accumulation and transport of microbial-size p...</td>\n",
       "      <td>Environmental Science</td>\n",
       "      <td>june152022_labels_envgeosoc</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>44397 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   text  \\\n",
       "0     The difficulty with this task lies in the fact...   \n",
       "1     The problem with rich annotations is that they...   \n",
       "2     As a consequence , when adapting existing meth...   \n",
       "3     The second problem of traditional word alignme...   \n",
       "4     The main drawback of these systems is that the...   \n",
       "...                                                 ...   \n",
       "1361  The area used for bioenergy crops (annual row ...   \n",
       "1421  Improper disposal of used polyethylene terepht...   \n",
       "1493  Airborne transmission of pathogenic aerosols v...   \n",
       "1513  Lead is one of the hazardous materials which i...   \n",
       "1548  Controlling airborne contamination is of major...   \n",
       "\n",
       "                            doi  \\\n",
       "0                           NaN   \n",
       "1                           NaN   \n",
       "2                           NaN   \n",
       "3                           NaN   \n",
       "4                           NaN   \n",
       "...                         ...   \n",
       "1361  10.1186/S13705-015-0067-3   \n",
       "1421  10.1007/s11367-012-0447-2   \n",
       "1493  10.4209/AAQR.2016.09.0400   \n",
       "1513   10.1186/2052-336X-12-103   \n",
       "1548    10.1186/1471-2334-11-58   \n",
       "\n",
       "                                            journalName  \\\n",
       "0                                                   NaN   \n",
       "1                                                   NaN   \n",
       "2                                                   NaN   \n",
       "3                                                   NaN   \n",
       "4                                                   NaN   \n",
       "...                                                 ...   \n",
       "1361                 Energy, Sustainability and Society   \n",
       "1421  The International Journal of Life Cycle Assess...   \n",
       "1493                   Aerosol and Air Quality Research   \n",
       "1513  Journal of Environmental Health Science and En...   \n",
       "1548                            BMC Infectious Diseases   \n",
       "\n",
       "                                                  title  \\\n",
       "0                                                   NaN   \n",
       "1                                                   NaN   \n",
       "2                                                   NaN   \n",
       "3                                                   NaN   \n",
       "4                                                   NaN   \n",
       "...                                                 ...   \n",
       "1361  Legume-based mixed intercropping systems may l...   \n",
       "1421  Comparative life cycle assessment and social l...   \n",
       "1493  Fluorescent Bioaerosol Particles Resulting fro...   \n",
       "1513  Covalently modified magnetite nanoparticles wi...   \n",
       "1548  Accumulation and transport of microbial-size p...   \n",
       "\n",
       "                     fields                       source  labels  \n",
       "0          Computer Science                acl_cambridge       1  \n",
       "1          Computer Science                acl_cambridge       1  \n",
       "2          Computer Science                acl_cambridge       1  \n",
       "3          Computer Science                acl_cambridge       1  \n",
       "4          Computer Science                acl_cambridge       1  \n",
       "...                     ...                          ...     ...  \n",
       "1361  Environmental Science  june152022_labels_envgeosoc       0  \n",
       "1421  Environmental Science  june152022_labels_envgeosoc       1  \n",
       "1493  Environmental Science  june152022_labels_envgeosoc       0  \n",
       "1513  Environmental Science  june152022_labels_envgeosoc       1  \n",
       "1548  Environmental Science  june152022_labels_envgeosoc       1  \n",
       "\n",
       "[44397 rows x 7 columns]"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_final"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aa48ebbe",
   "metadata": {},
   "source": [
    "## TRAIN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e909e75b",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "conda_python3",
   "language": "python",
   "name": "conda_python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}