feature_implementation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "98ce131d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GROBID server is up and running\n"
     ]
    }
   ],
   "source": [
    "from nltk.tokenize import word_tokenize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "5d1cbf5a",
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'parser' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m/var/folders/zn/vr53h0b96png6xmg4hd6yhb00000gn/T/ipykernel_22756/3812858972.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m# tables = read_pdf('/Users/revglue/study/main_work/my_working/dataset_pdf/E12-1072.pdf')\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;31m# print(tables)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mparsedPDF\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mparser\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfrom_file\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"/Users/revglue/study/main_work/my_working/dataset_pdf/E12-1072.pdf\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m \u001b[0;31m# print(parsedPDF['content'])\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mparsedPDF\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'metadata'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'parser' is not defined"
     ]
    }
   ],
   "source": [
    "# tables = read_pdf('/Users/revglue/study/main_work/my_working/dataset_pdf/E12-1072.pdf')\n",
    "# print(tables)\n",
    "parsedPDF = parser.from_file(\"/Users/revglue/study/main_work/my_working/dataset_pdf/E12-1072.pdf\")\n",
    "# print(parsedPDF['content'])\n",
    "print(parsedPDF['metadata'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3ad96cbe",
   "metadata": {},
   "outputs": [],
   "source": [
    "paper_file = open(\"/Users/revglue/study/main_work/my_working/testing_pdf/E12-1072.txt\",encoding='utf-8')\n",
    "paper_text = paper_file.read()\n",
    "paper_text = paper_text.strip() #removed whitespaces\n",
    "paper_text = paper_text.replace(\"-\\n\",\"\")\n",
    "paper_text = paper_text[paper_text.find('Abstract'):] #remove text before abstract\n",
    "# print(paper_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bc859f39",
   "metadata": {},
   "outputs": [],
   "source": [
    "paper_file = open(\"/Users/revglue/study/main_work/my_working/testing_pdf/E12-1072 copy.txt\",encoding='utf-8')\n",
    "paper_text = paper_file.read()\n",
    "paper_text = paper_text.strip() #removed whitespaces\n",
    "paper_text = paper_text.replace(\"-\\n\",\"\")\n",
    "paper_text = paper_text[paper_text.find('Abstract'):] #remove text before abstract\n",
    "print(paper_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4b064ff6",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "doc = nlp(paper_text)\n",
    "\n",
    "\n",
    "l = [sent.text for sent in doc.sents]\n",
    "\n",
    "for sentence in l:\n",
    "    if re.findall(r'\\(([^)]+)?(?:19|20)\\d{2}?([^)]+)?\\)', sentence):\n",
    "        print(sentence)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f02755c7",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"\\nfrom gensim\\n\")\n",
    "\n",
    "paper_sentence_list = tokenize.sent_tokenize(paper_text)\n",
    "# print(paper_sentence_list)\n",
    "\n",
    "# model = word2vec.Word2Vec([s.split() for s in paper_text], size=100, window=5, min_count=5, workers=4)\n",
    "# print(model)\n",
    "\n",
    "\n",
    "for sentence in paper_sentence_list:\n",
    "    tokenized_text = nltk.word_tokenize(sentence)\n",
    "    tokenized_text_new = []\n",
    "    for text in tokenized_text:\n",
    "        p_text = text.translate(str.maketrans('', '', string.punctuation))\n",
    "        tokenized_text_new.append(p_text)\n",
    "#     print(tokenized_text)\n",
    "#     text = sentence.translate(str.maketrans('', '', string.punctuation))\n",
    "#     text = BeautifulSoup(text, \"lxml\").text\n",
    "\n",
    "#     words = sentence.split()\n",
    "#     table = str.maketrans('', '', string.punctuation)\n",
    "#     stripped = [w.translate(table) for w in words]\n",
    "#     print(text)\n",
    "\n",
    "\n",
    "#     print(sentence.punctuation)\n",
    "#     table = sentence.maketrans('', '', string.punctuation)\n",
    "#     print(table)\n",
    "    \n",
    "#     tagged = nltk.pos_tag(tokenized_text)\n",
    "#     print(tagged)\n",
    "\n",
    "# print(paper_sentence_list)\n",
    "\n",
    "# print(\"\\nfrom nltk\\n\")\n",
    "# print(word_tokenize(paper_text))\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}