01-2_cleaning_depParsing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "d6167daa-f4d1-4dd4-87ba-e59081d4a913",
   "metadata": {},
   "source": [
    "## **Further Cleaning of the Fulltexts**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dd9da22b-67ae-4832-a1c9-b9ba2ae9ae8a",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "import re\n",
    "from tqdm.notebook import tqdm\n",
    "from datetime import datetime\n",
    "from collections import Counter\n",
    "from itertools import chain\n",
    "\n",
    "import scispacy\n",
    "import spacy\n",
    "from string import punctuation\n",
    "from nltk.tokenize import sent_tokenize, word_tokenize\n",
    "\n",
    "import multiprocessing as mp\n",
    "cpu_count = mp.cpu_count() - 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b03fd144-82b2-4e48-9bb0-5d30d7d4d57f",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# gibt Pfade zu den Texten zurück, in denen vorher schon \"virtual\"\n",
    "# die noch nicht gecleant wurden\n",
    "\n",
    "def get_paths(journal, basepath, outpath):\n",
    "    \n",
    "    textstodo = set([x for x in os.listdir(basepath + journal)])\n",
    "    textsdone = set([x for x in os.listdir(outpath + journal)])\n",
    "    textstodo = [basepath + journal + \"/\" + x for x in textstodo.difference(textsdone)]\n",
    "    \n",
    "    return textstodo\n",
    "\n",
    "def rejoin_sents(cleaned_sents):\n",
    "    \n",
    "    joined_sents = []\n",
    "    temp_sent = \"\"\n",
    "    for sent in cleaned_sents:\n",
    "        if sent[-1].isalpha():\n",
    "            sent += \".\"\n",
    "            joined_sents.append((temp_sent + sent).replace(\"  \", \" \"))\n",
    "            temp_sent = \"\"\n",
    "        elif sent[-1] == \".\":\n",
    "            joined_sents.append((temp_sent + sent).replace(\"  \", \" \"))\n",
    "            temp_sent = \"\"\n",
    "        else:\n",
    "            temp_sent += sent + \" \"\n",
    "            \n",
    "    if len(joined_sents) == 0:\n",
    "        joined_sents.append(temp_sent)\n",
    "            \n",
    "    return joined_sents\n",
    "\n",
    "def clean_paragraphs(paragraphs):\n",
    "    \n",
    "    cleaned_paragraphs = []\n",
    "    \n",
    "    for p in paragraphs:\n",
    "\n",
    "        p = p.replace(\"{\", \"(\") # wird oft falsch von OCR erkannt\n",
    "        p = p.replace(\"}\", \")\")\n",
    "        p = re.sub(\"\\(.*?\\)\", \"\", p) # alles in Klammern raus\n",
    "\n",
    "        # sentence segmentation\n",
    "        doc = nlp(p)\n",
    "\n",
    "        # cleaning\n",
    "        cleaned_sents = []\n",
    "        for sent in doc.sents:\n",
    "\n",
    "            sent_tokens = []\n",
    "            for token in sent:\n",
    "                # Nur Zahlen, rein alphabetische Wörter (da fliegt halt schlechtes OCR raus) und bestimmte Punktuationen\n",
    "                if token.is_digit or token.is_alpha or token.text in {\",\", \";\", \".\", \":\"}:\n",
    "                    sent_tokens.append(token.text_with_ws)\n",
    "\n",
    "            sent = \"\".join(sent_tokens)\n",
    "\n",
    "            if len(sent) > 10: cleaned_sents.append(sent)\n",
    "\n",
    "        if len(cleaned_sents) > 0:\n",
    "            out_p = \"\\n\".join(cleaned_sents)\n",
    "            out_p = out_p.replace(\"  \", \" \")\n",
    "            if out_p[0] == \" \": out_p = out_p[1:]\n",
    "            out_p = out_p.replace(\" . \", \". \")\n",
    "            out_p = out_p.replace(\" , \", \" \")\n",
    "            out_p = out_p.replace(\" , \", \", \")\n",
    "            out_p = out_p.replace(\"..\", \".\")\n",
    "            out_p = out_p.replace(\". .\", \"\")\n",
    "            out_p = out_p.replace(\" . \", \"\")\n",
    "            out_p = out_p.replace(\", ,\", \"\")\n",
    "            out_p = out_p.replace(\" , \", \"\")\n",
    "            out_p = out_p.replace(\" ; \", \"\")\n",
    "            out_p = out_p.replace(\" ; \", \";\")\n",
    "            out_p = out_p.replace(\" ;\", \"; \")\n",
    "            if out_p[-1].isalpha(): out_p += \".\"\n",
    "\n",
    "            cleaned_paragraphs.append(out_p)\n",
    "    \n",
    "    return cleaned_paragraphs\n",
    "\n",
    "# init clean\n",
    "# save to disc\n",
    "def innit(path):\n",
    "    \n",
    "    with open(path, \"r\") as f: \n",
    "        paragraphs = f.read().split(\"\\n\")\n",
    "        \n",
    "    cleaned_paragraphs = clean_paragraphs(paragraphs)\n",
    "    \n",
    "    # paragraphs seperated by double newline\n",
    "    with open(outpath + path.split(\"/\")[-2] + \"/\" + path.split(\"/\")[-1], \"w\") as f:\n",
    "        f.write(\"\\n\\n\".join(cleaned_paragraphs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b18b5d5f-7fe8-4271-a6b7-f5346631e3b6",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "nlp = spacy.load(\"en_core_sci_lg\")\n",
    "\n",
    "basepath = \"../../data/target_texts/\"\n",
    "outpath = \"../../data/cleaned_texts/\"\n",
    "\n",
    "journals = [\"rmp\", \"pr\", \"pra\", \"prb\", \"prc\", \"prd\", \"pre\", \"prl\"]\n",
    "\n",
    "for journal in journals:\n",
    "    \n",
    "    print(journal, \" - \", datetime.now().strftime(\"%H:%M:%S\"))\n",
    "    \n",
    "    if not os.path.isdir(outpath + journal):\n",
    "        os.mkdir(outpath + journal)\n",
    "    \n",
    "    paths = get_paths(journal, basepath, outpath)\n",
    "    \n",
    "    if __name__ == \"__main__\":\n",
    "        with mp.Pool(cpu_count) as pool:\n",
    "            pool.map(innit, paths)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8bc6547d-1d2d-41c7-bb11-d3d968ad2fac",
   "metadata": {},
   "source": [
    "### **Connect and Store Data in DataFrame**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2c7093b5-40fb-4855-b56f-e5a6b6a1965d",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "journals = [\"rmp\", \"pr\", \"pra\", \"prb\", \"prc\", \"prd\", \"pre\", \"prl\"]\n",
    "dfs = [pd.read_pickle(f\"../../data/combined_metadata/{journal}_metadata.pkl\") for journal in journals if os.path.isfile(f\"../../data/combined_metadata/{journal}_metadata.pkl\")]\n",
    "df = pd.concat(dfs, ignore_index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2bfd6ac9-5a7e-4e5e-a1e5-cc6a2306fa41",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "## check if cleaned texts still contain \"virtual\"\n",
    "outpath = \"../../data/cleaned_texts/\"\n",
    "target_dois = {}\n",
    "counter = 0\n",
    "\n",
    "for journal in tqdm(journals):\n",
    "    for path in os.listdir(outpath + journal):\n",
    "        with open(outpath + journal + \"/\" + path, \"r\") as f:\n",
    "            text = f.read()\n",
    "        search_text = text.lower()\n",
    "        if \"virtual\" in search_text:\n",
    "            doi = \"10.1103/\" + path[:-4]\n",
    "            target_dois[doi] = text\n",
    "        else:\n",
    "            counter += 1\n",
    "print(counter, \"Texte fallen noch raus.\")\n",
    "\n",
    "# merge together\n",
    "target_df = df.drop([\"volume\", \"issue\", \"bibcode\", \"aff\", \"arxiv_handle\", \"first_author\", \"database\", \"section\"], axis=1).copy()\n",
    "target_df = target_df.assign(text = pd.Series({**target_dois}))\n",
    "target_df = target_df[target_df.text.notna()] # nur Artikel mit volltext (dh mit virtual)\n",
    "target_df.to_pickle(\"../../data/cleaned_texts/cleaned_texts_df.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "571f9921-7480-4c08-991a-6f96bc860446",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "para_df = pd.read_pickle(\"../../data/cleaned_texts/filtered_paragraphs.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "61e0ec69-28e0-4150-a9a2-a4c881b4ac47",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "vdeps = [x for x in para_df.virtual_deps if x]\n",
    "Counter([x for x in chain.from_iterable(vdeps) if \"virtually\" in x]).most_common(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "40f36e1c-7372-45ff-9090-c2f9ed35bd5f",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def filter_adverbs(deps):\n",
    "    \n",
    "    keep = {\"virtually_excited_state\", \n",
    "        \"virtually_excited\", \n",
    "        \"virtually_energy\",\n",
    "        \"virtually_effect\",\n",
    "        \"virtually_state\",\n",
    "        \"virtually_bound_state\"}\n",
    "    \n",
    "    if not deps:\n",
    "        return None\n",
    "    \n",
    "    return_deps = []\n",
    "    for dep in deps:\n",
    "        if not target_word in dep or dep in keep:\n",
    "            return_deps.append(dep)\n",
    "    if len(return_deps) > 0:\n",
    "        return return_deps\n",
    "    else:\n",
    "        return None\n",
    "    \n",
    "# Merge dependencies into cleaned_texts df\n",
    "def get_list(llist):\n",
    "    temp = []\n",
    "    for x in llist:\n",
    "        if x:\n",
    "            for y in x:\n",
    "                temp.append(y)\n",
    "    return temp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "34b6c3ac-3d4f-4584-8fb6-cefbfaff3335",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "target_word = \"virtually\"\n",
    "para_df[\"virtual_deps\"] = para_df.virtual_deps.apply(filter_adverbs)\n",
    "\n",
    "para_df = para_df.loc[(para_df.virtual_deps.notna())].copy()\n",
    "para_df = para_df.reset_index(drop=True)\n",
    "\n",
    "para_df[\"length\"] = para_df.lemmas.apply(lambda x : len(list(chain.from_iterable(x))))\n",
    "para_df = para_df.loc[para_df.length > 4].copy()\n",
    "para_df = para_df.loc[para_df.length < 1500].copy()\n",
    "para_df = para_df.reset_index(drop=True)\n",
    "\n",
    "para_df[\"paragraph_id\"] = para_df.index\n",
    "\n",
    "para_df.to_pickle(\"../../data/cleaned_texts/filtered_paragraphs.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "26627a11-a2f1-4e7b-937c-287f77f32bbb",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_pickle(\"../../data/cleaned_texts/cleaned_texts_df.pkl\")\n",
    "df = df.drop([\"virtual_deps\"], axis=1)\n",
    "\n",
    "p_df = para_df.groupby(\"doi\").agg(\n",
    "    {\"virtual_deps\" : get_list})\n",
    "\n",
    "\n",
    "df = df.merge(p_df, how=\"left\", left_index=True, right_index=True)\n",
    "\n",
    "df.virtual_deps = df.virtual_deps.apply(lambda x: x if type(x) != float and len(x) > 0 else None)\n",
    "\n",
    "df.to_pickle(\"../../data/cleaned_texts/cleaned_texts_df.pkl\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "be3e4f50-eb3d-4607-85a6-27fb1f61e86e",
   "metadata": {},
   "source": [
    "## **Dependency Parsing and fixing of common OCR errors**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "da1a9511-983c-4bdc-bc5f-76483e90c199",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "df = pd.read_pickle(\"../../data/cleaned_texts/cleaned_texts_df.pkl\")\n",
    "df = df[df.virtual_deps.notna()].copy()\n",
    "df = df.drop([\"virtual_deps\"], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "51d0485d-f7e3-4d46-8b6d-bb8464d3aca1",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Fehler bei denen OCR die Leerzeichen nicht erkannt hat lösen wie virtualparticle, etc. lösen,\n",
    "def fix_ocr(text):\n",
    "    text = text.lower()\n",
    "    sents = sent_tokenize(text)\n",
    "    new_sents = []\n",
    "    for sent in sents:\n",
    "        words = word_tokenize(sent)\n",
    "        new_words = []\n",
    "        for word in words:\n",
    "            if \"virtual\" in word and word not in {\"virtual\", \"virtually\", \"virtuality\", \"virtualities\"}:\n",
    "                if \"virtualities\" in word:\n",
    "                    new_words.append(\"virtualities\")\n",
    "                    new_words.append(word.split(\"virtualities\")[1])\n",
    "                elif \"virtuality\" in word:\n",
    "                    new_words.append(\"virtuality\")\n",
    "                    new_words.append(word.split(\"virtuality\")[1])\n",
    "                elif \"virtually\" in word:\n",
    "                    new_words.append(\"virtually\")\n",
    "                    new_words.append(word.split(\"virtually\")[1])\n",
    "                else:\n",
    "                    new_words.append(\"virtual\")\n",
    "                    new_words.append(word.split(\"virtual\")[1])\n",
    "            else:\n",
    "                new_words.append(word)\n",
    "        new_sents.append(\" \".join(new_words))\n",
    "    new_text = \" \".join(new_sents)\n",
    "    new_text = new_text.replace(\" .\", \".\")\n",
    "    new_text = new_text.replace(\" ,\", \",\")\n",
    "    new_text = new_text.replace(\" ;\", \";\")\n",
    "    new_text = new_text.replace(\" :\", \":\")\n",
    "    return new_text\n",
    "    \n",
    "def depparsing(doi):\n",
    "    \n",
    "    deps = []\n",
    "    doc = nlp(df.at[doi, \"text\"])\n",
    "    for sent in doc.sents:\n",
    "        for token in sent:\n",
    "            if \"virtual\" in token.text:\n",
    "                temp_dep = [token.text]\n",
    "                if token.text == \"virtuality\" or token.text == \"virtualities\":\n",
    "                    for child in token.children:\n",
    "                        temp_dep.append(child.text)\n",
    "                else:\n",
    "                    head = token.head\n",
    "                    while head.pos_ != \"NOUN\":\n",
    "                        temp_dep.append(head.text)\n",
    "                        if head == head.head: # prevent infinity loop\n",
    "                            break\n",
    "                        head = head.head\n",
    "                    temp_dep.append(head.text)\n",
    "                deps.append(temp_dep)\n",
    "    return deps\n",
    "        \n",
    "    \n",
    "def join_deps(deps):\n",
    "    temp = []\n",
    "    # fix error for adverbs\n",
    "    for dep in deps:\n",
    "        if len(dep) >= 2:\n",
    "            if dep[-1] == dep[-2]:\n",
    "                dep = dep[:-1]\n",
    "        if len(dep) > 0:\n",
    "            temp.append(\"_\".join(dep))\n",
    "    if len(temp) > 0:\n",
    "        return temp\n",
    "    else:\n",
    "        return None\n",
    "    \n",
    "    \n",
    "# cleans dependencies to merge plural / singular\n",
    "def clean_a_bit(deps):\n",
    "\n",
    "    skip_set = {\"nucleus\", \"axis\", \"indices\", \"hypothesis\"}\n",
    "    return_deps = []\n",
    "    for dep in deps:\n",
    "        temp_dep = []\n",
    "        for word in dep:\n",
    "            if word[-1] == \"s\":\n",
    "                if word in skip_set:\n",
    "                    temp_dep.append(word)\n",
    "                    continue\n",
    "                if word[-2:] == \"ss\":\n",
    "                    temp_dep.append(word)\n",
    "                    continue\n",
    "                if word[-3:] == \"ies\":\n",
    "                    temp_dep.append(word[:-3] + \"y\")\n",
    "                    continue\n",
    "                if word[-4:] == \"sses\":\n",
    "                    temp_dep.append(word[:-2])\n",
    "                    continue\n",
    "                temp_dep.append(word[:-1])\n",
    "            else:\n",
    "                temp_dep.append(word)\n",
    "\n",
    "        return_deps.append(temp_dep)\n",
    "    return return_deps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d4642f92-0d9b-4129-a915-14cf8d02093a",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Clean OCR errors\n",
    "\n",
    "for index in tqdm(df.index):\n",
    "    \n",
    "    df.at[index, \"text\"] = fix_ocr(df.at[index, \"text\"])\n",
    "    \n",
    "df.to_pickle(\"../../data/cleaned_texts/virtual_fulltexts.df\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "54b86a4a-fd1c-42dd-b867-b99e4f0d8d15",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Dependency Parsing\n",
    "\n",
    "df = pd.read_pickle(\"../../data/cleaned_texts/virtual_fulltexts.df\")\n",
    "\n",
    "nlp = spacy.load(\"en_core_sci_lg\")\n",
    "\n",
    "df[\"deps\"] = None\n",
    "df[\"cleaned_deps\"] = None\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    with mp.Pool(cpu_count) as pool:\n",
    "        dep_list = pool.map(depparsing, df.index)\n",
    "\n",
    "for doi, deps in zip(df.index, dep_list):\n",
    "    \n",
    "    cleaned_deps = clean_a_bit(deps)\n",
    "    df.at[doi, \"deps\"] = join_deps(deps)\n",
    "    df.at[doi, \"cleaned_deps\"] = join_deps(cleaned_deps)\n",
    "    \n",
    "df.to_pickle(\"../../data/cleaned_texts/virtual_fulltexts.df\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}