summarize.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "467b2c07",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to\n",
      "[nltk_data]     C:\\Users\\qmok9\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n",
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     C:\\Users\\qmok9\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n",
      "[nltk_data] Downloading package wordnet to\n",
      "[nltk_data]     C:\\Users\\qmok9\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package wordnet is already up-to-date!\n",
      "[nltk_data] Downloading package omw-1.4 to\n",
      "[nltk_data]     C:\\Users\\qmok9\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package omw-1.4 is already up-to-date!\n",
      "[nltk_data] Downloading package inaugural to\n",
      "[nltk_data]     C:\\Users\\qmok9\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Unzipping corpora\\inaugural.zip.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import re\n",
    "import math\n",
    "import nltk\n",
    "\n",
    "nltk.download('punkt')\n",
    "nltk.download('stopwords')\n",
    "nltk.download('wordnet')\n",
    "nltk.download('omw-1.4')\n",
    "nltk.download('inaugural')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "52076f71",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_word_frequencies(text):\n",
    "    \"\"\"Get word frequencies from a text.\n",
    "    \n",
    "    Parameters\n",
    "    ----------\n",
    "    text : str\n",
    "        The text to be analyzed\n",
    "        \n",
    "    Returns\n",
    "    -------\n",
    "    result : dict\n",
    "        A dictionary with relative word frequencies\n",
    "    \"\"\"\n",
    "    clean_text = re.sub('[^A-Za-z]+', ' ', text)\n",
    "    tokenized = nltk.word_tokenize(clean_text.lower())\n",
    "    stopwords = nltk.corpus.stopwords.words('english')\n",
    "\n",
    "    result = {}\n",
    "    for word in tokenized:\n",
    "        if word not in stopwords:\n",
    "            if word not in result.keys():\n",
    "                result[word] = 1\n",
    "            else:\n",
    "                result[word] += 1\n",
    "    \n",
    "    max_frequncy = max(result.values())\n",
    "    for word in result.keys():\n",
    "        result[word] = (result[word]/max_frequncy)\n",
    "        \n",
    "    return result\n",
    "\n",
    "\n",
    "def get_sentence_scores(text, word_frequencies):\n",
    "    \"\"\"Get all sentence scores based on word frequencies\n",
    "    \n",
    "    Parameters\n",
    "    ----------\n",
    "    text : str\n",
    "        The text to be analyzed\n",
    "        \n",
    "    word_frequencies : dict\n",
    "        A dictionary with relative word frequencies    \n",
    "        \n",
    "    Returns\n",
    "    -------\n",
    "    result : dict\n",
    "        A dictionary containing a score for every sentence\n",
    "    \"\"\"\n",
    "    sentences = nltk.sent_tokenize(text)\n",
    "    result = {}\n",
    "    for sentence in sentences:\n",
    "        words = nltk.word_tokenize(sentence.lower())\n",
    "        for word in words:\n",
    "            if word in word_frequencies.keys():\n",
    "                if sentence not in result.keys():\n",
    "                    result[sentence] = word_frequencies[word]\n",
    "                else:\n",
    "                    result[sentence] += word_frequencies[word]\n",
    "                    \n",
    "    return result\n",
    "\n",
    "\n",
    "def get_summary(text, max_length):\n",
    "    \"\"\"Get a summary of given text\n",
    "    \n",
    "    Parameters\n",
    "    ----------\n",
    "    text : str\n",
    "        The text to be summarized\n",
    "        \n",
    "    max_length : int\n",
    "        Maximum length of the summary measured in number of tokens  \n",
    "        \n",
    "    Returns\n",
    "    -------\n",
    "    result : str\n",
    "        The summarized text\n",
    "    \"\"\"\n",
    "    sentence_scores = get_sentence_scores(text, get_word_frequencies(text))\n",
    "    result = []\n",
    "    current_length = 0\n",
    "    while current_length <= max_length and len(sentence_scores) > 0:\n",
    "        summary_sent = max(sentence_scores, key=sentence_scores.get)\n",
    "        sent_length = len(nltk.word_tokenize(summary_sent.lower()))\n",
    "        if current_length != 0 and current_length + sent_length > max_length:\n",
    "            break\n",
    "        else:\n",
    "            result.append(summary_sent)\n",
    "            current_length += sent_length\n",
    "            sentence_scores.pop(summary_sent)\n",
    "    \n",
    "    return ' '.join(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "id": "244222e4",
   "metadata": {},
   "outputs": [],
   "source": [
    "def slice_text(text, max_length):\n",
    "    \"\"\"Slice the given text into smaller parts regarding max_length\n",
    "    \n",
    "    Parameters\n",
    "    ----------\n",
    "    text : str\n",
    "        The text to be sliced\n",
    "        \n",
    "    max_length : int\n",
    "        Maximum length of each slice measured in number of tokens  \n",
    "        \n",
    "    Returns\n",
    "    -------\n",
    "    result : list\n",
    "        A list with text slices\n",
    "    \"\"\"\n",
    "    sentences = nltk.sent_tokenize(text)\n",
    "    result = []\n",
    "    temp = []\n",
    "    current_length = 0\n",
    "    for sentence in sentences:\n",
    "        sent_length = len(nltk.word_tokenize(sentence))\n",
    "        if current_length != 0 and current_length + sent_length > max_length:\n",
    "            result.append(' '.join(temp))\n",
    "            current_length = 0\n",
    "            temp = []\n",
    "        temp.append(sentence)\n",
    "        current_length += sent_length\n",
    "    if len(temp) > 0:\n",
    "        result.append(' '.join(temp))\n",
    "    return result\n",
    "\n",
    "def summarize_text(text, max_length):\n",
    "    \"\"\"Summarize a long text hierarchically by first summarizing slices\n",
    "    of the text, then putting the summaries together\n",
    "    \n",
    "    Parameters\n",
    "    ----------\n",
    "    text : str\n",
    "        The text to be summarized\n",
    "        \n",
    "    max_length : int\n",
    "        Maximum length of the summary measured in number of tokens  \n",
    "        \n",
    "    Returns\n",
    "    -------\n",
    "    result : str\n",
    "        The summarized text\n",
    "    \"\"\"\n",
    "    text_length = len(nltk.word_tokenize(text))\n",
    "    if text_length < max_length:\n",
    "        return text\n",
    "    else:\n",
    "        slice_length = math.ceil(text_length / math.ceil(text_length / max_length))\n",
    "        summary_length = slice_length / 2\n",
    "        sliced_texts = slice_text(text, slice_length)\n",
    "        summaries = []\n",
    "        for sliced_text in sliced_texts:\n",
    "            summaries.append(get_summary(sliced_text, summary_length))\n",
    "        result = ' '.join(summaries)\n",
    "        if len(nltk.word_tokenize(result)) <= max_length:\n",
    "            return result\n",
    "        else:\n",
    "            return summarize_text(result, max_length)\n",
    "\n",
    "def write_document(text, title):\n",
    "    f = open(title + \".txt\", \"w\")\n",
    "    f.write(text)\n",
    "    f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "c35d09a4",
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.corpus import inaugural\n",
    "CONTEXT_WINDOW_LENGTH = 400 #reduced for testing\n",
    "text1 = inaugural.raw('1789-Washington.txt')\n",
    "text2 = inaugural.raw('1793-Washington.txt')\n",
    "write_document(summarize_text(text1, CONTEXT_WINDOW_LENGTH), \"summary1\")\n",
    "write_document(summarize_text(text2, CONTEXT_WINDOW_LENGTH), \"summary2\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}