MS4.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"provenance":[],"gpuType":"T4","machine_shape":"hm"},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"},"accelerator":"GPU"},"cells":[{"cell_type":"markdown","source":["This IPYNB file provides a comprehensive exploration of the impact of sentiment fine-tuning on text summarization, utilizing the kindle_reviews.csv dataset. The process includes loading and splitting the data into training and testing sets, applying Hugging Face's sentiment classifier to categorize each review, and fine-tuning a T5 model based on these sentiment classifications. The notebook then generates summaries for the test dataset, offering insights into how sentiment-based fine-tuning alters the summarization process compared to traditional methods, with parameters adjustable to fit various runtime and analysis requirements.\n","\n","\n","\n","\n","\n"],"metadata":{"id":"dWusHezLCNGh"}},{"cell_type":"code","execution_count":1,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"4kjPnyXO_msI","executionInfo":{"status":"ok","timestamp":1703285547663,"user_tz":300,"elapsed":8792,"user":{"displayName":"Madelyn Dempsey","userId":"12178507243338002500"}},"outputId":"10c39bb3-7be8-41ef-e3d5-edae673b4beb"},"outputs":[{"output_type":"stream","name":"stdout","text":["Requirement already satisfied: transformers in /usr/local/lib/python3.10/dist-packages (4.35.2)\n","Requirement already satisfied: datasets in /usr/local/lib/python3.10/dist-packages (2.16.0)\n","Requirement already satisfied: rouge-score in /usr/local/lib/python3.10/dist-packages (0.1.2)\n","Requirement already satisfied: torch in /usr/local/lib/python3.10/dist-packages (2.1.0+cu121)\n","Requirement already satisfied: sentencepiece in /usr/local/lib/python3.10/dist-packages (0.1.99)\n","Requirement already satisfied: accelerate in /usr/local/lib/python3.10/dist-packages (0.25.0)\n","Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers) (3.13.1)\n","Requirement already satisfied: huggingface-hub<1.0,>=0.16.4 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.19.4)\n","Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (1.23.5)\n","Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers) (23.2)\n","Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (6.0.1)\n","Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (2023.6.3)\n","Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers) (2.31.0)\n","Requirement already satisfied: tokenizers<0.19,>=0.14 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.15.0)\n","Requirement already satisfied: safetensors>=0.3.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.4.1)\n","Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers) (4.66.1)\n","Requirement already satisfied: pyarrow>=8.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (10.0.1)\n","Requirement already satisfied: pyarrow-hotfix in /usr/local/lib/python3.10/dist-packages (from datasets) (0.6)\n","Requirement already satisfied: dill<0.3.8,>=0.3.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (0.3.7)\n","Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from datasets) (1.5.3)\n","Requirement already satisfied: xxhash in /usr/local/lib/python3.10/dist-packages (from datasets) (3.4.1)\n","Requirement already satisfied: multiprocess in /usr/local/lib/python3.10/dist-packages (from datasets) (0.70.15)\n","Requirement already satisfied: fsspec[http]<=2023.10.0,>=2023.1.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (2023.6.0)\n","Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets) (3.9.1)\n","Requirement already satisfied: absl-py in /usr/local/lib/python3.10/dist-packages (from rouge-score) (1.4.0)\n","Requirement already satisfied: nltk in /usr/local/lib/python3.10/dist-packages (from rouge-score) (3.8.1)\n","Requirement already satisfied: six>=1.14.0 in /usr/local/lib/python3.10/dist-packages (from rouge-score) (1.16.0)\n","Requirement already satisfied: typing-extensions in /usr/local/lib/python3.10/dist-packages (from torch) (4.5.0)\n","Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch) (1.12)\n","Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch) (3.2.1)\n","Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch) (3.1.2)\n","Requirement already satisfied: triton==2.1.0 in /usr/local/lib/python3.10/dist-packages (from torch) (2.1.0)\n","Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from accelerate) (5.9.5)\n","Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (23.1.0)\n","Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (6.0.4)\n","Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.9.4)\n","Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.4.1)\n","Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.3.1)\n","Requirement already satisfied: async-timeout<5.0,>=4.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (4.0.3)\n","Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.3.2)\n","Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.6)\n","Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2.0.7)\n","Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2023.11.17)\n","Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch) (2.1.3)\n","Requirement already satisfied: click in /usr/local/lib/python3.10/dist-packages (from nltk->rouge-score) (8.1.7)\n","Requirement already satisfied: joblib in /usr/local/lib/python3.10/dist-packages (from nltk->rouge-score) (1.3.2)\n","Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2.8.2)\n","Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2023.3.post1)\n","Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch) (1.3.0)\n"]}],"source":["!pip install transformers datasets rouge-score torch sentencepiece accelerate"]},{"cell_type":"code","source":["import pandas as pd\n","import csv\n","from transformers import T5Tokenizer, T5ForConditionalGeneration, pipeline\n","from datasets import load_dataset, load_metric\n","from torch.utils.data import Dataset, DataLoader\n","\n","df = pd.read_csv(\"kindle_reviews.csv\", error_bad_lines=False, nrows=1000)\n","df.head()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":627},"id":"_SCu9BLx_vXq","executionInfo":{"status":"ok","timestamp":1703286521501,"user_tz":300,"elapsed":360,"user":{"displayName":"Madelyn Dempsey","userId":"12178507243338002500"}},"outputId":"54dbaeb9-040b-4b6b-b1e3-d73106af7971"},"execution_count":8,"outputs":[{"output_type":"stream","name":"stderr","text":["<ipython-input-8-d1813653063f>:7: FutureWarning: The error_bad_lines argument has been deprecated and will be removed in a future version. Use on_bad_lines in the future.\n","\n","\n","  df = pd.read_csv(\"kindle_reviews.csv\", error_bad_lines=False, nrows=1000)\n"]},{"output_type":"execute_result","data":{"text/plain":["   Unnamed: 0        asin helpful  overall  \\\n","0           0  B000F83SZQ  [0, 0]        5   \n","1           1  B000F83SZQ  [2, 2]        4   \n","2           2  B000F83SZQ  [2, 2]        4   \n","3           3  B000F83SZQ  [1, 1]        5   \n","4           4  B000F83SZQ  [0, 1]        4   \n","\n","                                          reviewText   reviewTime  \\\n","0  I enjoy vintage books and movies so I enjoyed ...   05 5, 2014   \n","1  This book is a reissue of an old one; the auth...   01 6, 2014   \n","2  This was a fairly interesting read.  It had ol...   04 4, 2014   \n","3  I'd never read any of the Amy Brewster mysteri...  02 19, 2014   \n","4  If you like period pieces - clothing, lingo, y...  03 19, 2014   \n","\n","       reviewerID                         reviewerName             summary  \\\n","0  A1F6404F1VG29J                           Avidreader  Nice vintage story   \n","1   AN0N05A9LIJEQ                             critters        Different...   \n","2   A795DMNCJILA6                                  dot               Oldie   \n","3  A1FV0SX13TWVXQ  Elaine H. Turley \"Montana Songbird\"  I really liked it.   \n","4  A3SPTOKDG7WBLN                   Father Dowling Fan      Period Mystery   \n","\n","   unixReviewTime  \n","0      1399248000  \n","1      1388966400  \n","2      1396569600  \n","3      1392768000  \n","4      1395187200  "],"text/html":["\n","  <div id=\"df-9dd2273c-06df-4a98-a456-a3cfcc2b886a\" class=\"colab-df-container\">\n","    <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>Unnamed: 0</th>\n","      <th>asin</th>\n","      <th>helpful</th>\n","      <th>overall</th>\n","      <th>reviewText</th>\n","      <th>reviewTime</th>\n","      <th>reviewerID</th>\n","      <th>reviewerName</th>\n","      <th>summary</th>\n","      <th>unixReviewTime</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>0</td>\n","      <td>B000F83SZQ</td>\n","      <td>[0, 0]</td>\n","      <td>5</td>\n","      <td>I enjoy vintage books and movies so I enjoyed ...</td>\n","      <td>05 5, 2014</td>\n","      <td>A1F6404F1VG29J</td>\n","      <td>Avidreader</td>\n","      <td>Nice vintage story</td>\n","      <td>1399248000</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>1</td>\n","      <td>B000F83SZQ</td>\n","      <td>[2, 2]</td>\n","      <td>4</td>\n","      <td>This book is a reissue of an old one; the auth...</td>\n","      <td>01 6, 2014</td>\n","      <td>AN0N05A9LIJEQ</td>\n","      <td>critters</td>\n","      <td>Different...</td>\n","      <td>1388966400</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>2</td>\n","      <td>B000F83SZQ</td>\n","      <td>[2, 2]</td>\n","      <td>4</td>\n","      <td>This was a fairly interesting read.  It had ol...</td>\n","      <td>04 4, 2014</td>\n","      <td>A795DMNCJILA6</td>\n","      <td>dot</td>\n","      <td>Oldie</td>\n","      <td>1396569600</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>3</td>\n","      <td>B000F83SZQ</td>\n","      <td>[1, 1]</td>\n","      <td>5</td>\n","      <td>I'd never read any of the Amy Brewster mysteri...</td>\n","      <td>02 19, 2014</td>\n","      <td>A1FV0SX13TWVXQ</td>\n","      <td>Elaine H. Turley \"Montana Songbird\"</td>\n","      <td>I really liked it.</td>\n","      <td>1392768000</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>4</td>\n","      <td>B000F83SZQ</td>\n","      <td>[0, 1]</td>\n","      <td>4</td>\n","      <td>If you like period pieces - clothing, lingo, y...</td>\n","      <td>03 19, 2014</td>\n","      <td>A3SPTOKDG7WBLN</td>\n","      <td>Father Dowling Fan</td>\n","      <td>Period Mystery</td>\n","      <td>1395187200</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>\n","    <div class=\"colab-df-buttons\">\n","\n","  <div class=\"colab-df-container\">\n","    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-9dd2273c-06df-4a98-a456-a3cfcc2b886a')\"\n","            title=\"Convert this dataframe to an interactive table.\"\n","            style=\"display:none;\">\n","\n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n","    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n","  </svg>\n","    </button>\n","\n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    .colab-df-buttons div {\n","      margin-bottom: 4px;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","    <script>\n","      const buttonEl =\n","        document.querySelector('#df-9dd2273c-06df-4a98-a456-a3cfcc2b886a button.colab-df-convert');\n","      buttonEl.style.display =\n","        google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","      async function convertToInteractive(key) {\n","        const element = document.querySelector('#df-9dd2273c-06df-4a98-a456-a3cfcc2b886a');\n","        const dataTable =\n","          await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                    [key], {});\n","        if (!dataTable) return;\n","\n","        const docLinkHtml = 'Like what you see? Visit the ' +\n","          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","          + ' to learn more about interactive tables.';\n","        element.innerHTML = '';\n","        dataTable['output_type'] = 'display_data';\n","        await google.colab.output.renderOutput(dataTable, element);\n","        const docLink = document.createElement('div');\n","        docLink.innerHTML = docLinkHtml;\n","        element.appendChild(docLink);\n","      }\n","    </script>\n","  </div>\n","\n","\n","<div id=\"df-fef216b9-f0eb-4d4e-8b0d-78a7bb2f8b47\">\n","  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-fef216b9-f0eb-4d4e-8b0d-78a7bb2f8b47')\"\n","            title=\"Suggest charts\"\n","            style=\"display:none;\">\n","\n","<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","     width=\"24px\">\n","    <g>\n","        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n","    </g>\n","</svg>\n","  </button>\n","\n","<style>\n","  .colab-df-quickchart {\n","      --bg-color: #E8F0FE;\n","      --fill-color: #1967D2;\n","      --hover-bg-color: #E2EBFA;\n","      --hover-fill-color: #174EA6;\n","      --disabled-fill-color: #AAA;\n","      --disabled-bg-color: #DDD;\n","  }\n","\n","  [theme=dark] .colab-df-quickchart {\n","      --bg-color: #3B4455;\n","      --fill-color: #D2E3FC;\n","      --hover-bg-color: #434B5C;\n","      --hover-fill-color: #FFFFFF;\n","      --disabled-bg-color: #3B4455;\n","      --disabled-fill-color: #666;\n","  }\n","\n","  .colab-df-quickchart {\n","    background-color: var(--bg-color);\n","    border: none;\n","    border-radius: 50%;\n","    cursor: pointer;\n","    display: none;\n","    fill: var(--fill-color);\n","    height: 32px;\n","    padding: 0;\n","    width: 32px;\n","  }\n","\n","  .colab-df-quickchart:hover {\n","    background-color: var(--hover-bg-color);\n","    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n","    fill: var(--button-hover-fill-color);\n","  }\n","\n","  .colab-df-quickchart-complete:disabled,\n","  .colab-df-quickchart-complete:disabled:hover {\n","    background-color: var(--disabled-bg-color);\n","    fill: var(--disabled-fill-color);\n","    box-shadow: none;\n","  }\n","\n","  .colab-df-spinner {\n","    border: 2px solid var(--fill-color);\n","    border-color: transparent;\n","    border-bottom-color: var(--fill-color);\n","    animation:\n","      spin 1s steps(1) infinite;\n","  }\n","\n","  @keyframes spin {\n","    0% {\n","      border-color: transparent;\n","      border-bottom-color: var(--fill-color);\n","      border-left-color: var(--fill-color);\n","    }\n","    20% {\n","      border-color: transparent;\n","      border-left-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","    }\n","    30% {\n","      border-color: transparent;\n","      border-left-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","      border-right-color: var(--fill-color);\n","    }\n","    40% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","    }\n","    60% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","    }\n","    80% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","      border-bottom-color: var(--fill-color);\n","    }\n","    90% {\n","      border-color: transparent;\n","      border-bottom-color: var(--fill-color);\n","    }\n","  }\n","</style>\n","\n","  <script>\n","    async function quickchart(key) {\n","      const quickchartButtonEl =\n","        document.querySelector('#' + key + ' button');\n","      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n","      quickchartButtonEl.classList.add('colab-df-spinner');\n","      try {\n","        const charts = await google.colab.kernel.invokeFunction(\n","            'suggestCharts', [key], {});\n","      } catch (error) {\n","        console.error('Error during call to suggestCharts:', error);\n","      }\n","      quickchartButtonEl.classList.remove('colab-df-spinner');\n","      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n","    }\n","    (() => {\n","      let quickchartButtonEl =\n","        document.querySelector('#df-fef216b9-f0eb-4d4e-8b0d-78a7bb2f8b47 button');\n","      quickchartButtonEl.style.display =\n","        google.colab.kernel.accessAllowed ? 'block' : 'none';\n","    })();\n","  </script>\n","</div>\n","    </div>\n","  </div>\n"]},"metadata":{},"execution_count":8}]},{"cell_type":"code","source":["# trying different sentiment analysis pipelines\n","financial_news_pipeline = pipeline(model=\"mrm8488/distilroberta-finetuned-financial-news-sentiment-analysis\")\n","general_news_pipeline = pipeline(model=\"shashanksrinath/News_Sentiment_Analysis\")\n","twitter_pipeline = pipeline(model=\"cardiffnlp/twitter-roberta-base-sentiment\")\n","sentiment = pipeline('sentiment-analysis')"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"eLcFZd0W_xTq","executionInfo":{"status":"ok","timestamp":1703286537707,"user_tz":300,"elapsed":14876,"user":{"displayName":"Madelyn Dempsey","userId":"12178507243338002500"}},"outputId":"fe8544f7-6d5e-43db-fad5-6b881a1e8e72"},"execution_count":9,"outputs":[{"output_type":"stream","name":"stderr","text":["No model was supplied, defaulted to distilbert-base-uncased-finetuned-sst-2-english and revision af0f99b (https://huggingface.co/distilbert-base-uncased-finetuned-sst-2-english).\n","Using a pipeline without specifying a model name and revision in production is not recommended.\n"]}]},{"cell_type":"code","source":["error_count = 0\n","num_pos = 0\n","num_neg = 0\n","tot = 0\n","def get_sentiment(text):\n","    global error_count, num_pos, num_neg, tot\n","    try:\n","        result = sentiment(text)\n","        label = result[0]['label']\n","        tot += 1\n","        num_pos += 1 if label == 'POSITIVE' else 0\n","        num_neg += 1 if label == 'NEGATIVE' else 0\n","        return label\n","    except Exception as e:\n","        # print(f\"Error: {str(e)}. Text: {text}.\")\n","        error_count += 1\n","        return None\n","\n","df['sentiment'] = df['reviewText'].apply(get_sentiment)\n","#df['summary_sentiment'] = df['summary'].apply(get_sentiment)\n","\n","print(df['sentiment'])\n","print(f'{error_count} rows threw an exception.')\n","print(f\"num positive = {num_pos}\\n Num negative = {num_neg}, tot={tot}\")\n","print(f\"Percent positive input text = {100 * num_pos / tot}\\n Percent negative input text = {100 * num_neg / tot}\")\n","#print(f\"Percent positive gold-summary= {100 * num_pos / tot}\\n Percent negative gold-summary= {100 * num_neg / tot}\")\n","\n"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"EeNzdD4K3Dnd","executionInfo":{"status":"ok","timestamp":1703287698891,"user_tz":300,"elapsed":196603,"user":{"displayName":"Madelyn Dempsey","userId":"12178507243338002500"}},"outputId":"a60e9771-58d9-450c-e454-16d631f9a160"},"execution_count":13,"outputs":[{"output_type":"stream","name":"stdout","text":["0      POSITIVE\n","1      POSITIVE\n","2      POSITIVE\n","3      POSITIVE\n","4      POSITIVE\n","         ...   \n","995    POSITIVE\n","996    POSITIVE\n","997    POSITIVE\n","998    NEGATIVE\n","999    POSITIVE\n","Name: sentiment, Length: 1000, dtype: object\n","31 rows threw an exception.\n","num positive = 753\n"," Num negative = 216, tot=969\n","Percent positive input text = 77.70897832817337\n"," Percent negative input text = 22.291021671826627\n"]}]},{"cell_type":"code","source":["df_positive = df[df['sentiment'] == 'POSITIVE'].drop(columns=['sentiment'])\n","df_negative = df[df['sentiment'] == 'NEGATIVE'].drop(columns=['sentiment'])\n","df_neutral = df[df['sentiment'] == 'NEUTRAL'].drop(columns=['sentiment'])\n","\n","# df_summary_positive = df[df['summary_sentiment'] == 'POSITIVE'].drop(columns=['summary_sentiment'])\n","# df_summary_negative = df[df['summary_sentiment'] == 'NEGATIVE'].drop(columns=['summary_sentiment'])\n","# df_summary_neutral = df[df['summary_sentiment'] == 'NEUTRAL'].drop(columns=['summary_sentiment'])\n","\n","print(f'There are {len(df_positive)} positive rows, {len(df_negative)} negative rows, and {len(df_neutral)} neutral rows when measuring sentiment of the input review text.')\n","#print(f'There are {len(df_summary_positive)} positive rows, {len(df_summary_negative)} negative rows, and {len(df_summary_neutral)} neutral rows when measuring sentiment of the gold standard summaries.')"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"yhWlZgiP_-xt","executionInfo":{"status":"ok","timestamp":1703287698892,"user_tz":300,"elapsed":38,"user":{"displayName":"Madelyn Dempsey","userId":"12178507243338002500"}},"outputId":"74377de2-89c6-4da2-f7c0-b16ca8ac1e24"},"execution_count":14,"outputs":[{"output_type":"stream","name":"stdout","text":["There are 753 positive rows, 216 negative rows, and 0 neutral rows when measuring sentiment of the input review text.\n"]}]},{"cell_type":"code","source":["class KindleReviewDataset(Dataset):\n","    def __init__(self, tokenizer, data, max_length=512):\n","        self.tokenizer = tokenizer\n","        self.data = data\n","        self.max_length = max_length\n","\n","    def __len__(self):\n","        return len(self.data)\n","\n","    def __getitem__(self, idx):\n","        item = self.data.iloc[idx]\n","        text = item['reviewText']\n","        summary = item['summary']\n","        inputs = self.tokenizer.encode_plus(\n","            text, max_length=self.max_length, truncation=True, padding='max_length', return_tensors='pt')\n","        targets = self.tokenizer.encode_plus(\n","            summary, max_length=self.max_length, truncation=True, padding='max_length', return_tensors='pt')\n","        return {\n","            'input_ids': inputs['input_ids'].flatten(),\n","            'attention_mask': inputs['attention_mask'].flatten(),\n","            'labels': targets['input_ids'].flatten()\n","        }"],"metadata":{"id":"jUsrC9MTDAX5","executionInfo":{"status":"aborted","timestamp":1703285701971,"user_tz":300,"elapsed":5,"user":{"displayName":"Madelyn Dempsey","userId":"12178507243338002500"}}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["r = 0.95  # Fraction of data for training\n","\n","# Function to split data\n","def split_data(dataframe, train_frac):\n","    train_size = int(len(dataframe) * train_frac)\n","    train_df = dataframe[:train_size]\n","    test_df = dataframe[train_size:]\n","    return train_df, test_df\n","\n","# Splitting the datasets\n","df_positive = df[df['sentiment'] == 'POSITIVE'].drop(columns=['sentiment'])\n","df_negative = df[df['sentiment'] == 'NEGATIVE'].drop(columns=['sentiment'])\n","df_full = df.drop(columns=['sentiment'])  # Full dataset\n","\n","print(len(df_positive))\n","print(len(df_negative))\n","print(len(df_full))\n","\n","train_df_full, test_df_full = split_data(df_full, r)\n","train_df_positive, test_df_positive = split_data(df_positive, r)\n","train_df_negative, test_df_negative = split_data(df_negative, r)\n"],"metadata":{"id":"Sma-JEQy5jmK","executionInfo":{"status":"aborted","timestamp":1703285701971,"user_tz":300,"elapsed":4,"user":{"displayName":"Madelyn Dempsey","userId":"12178507243338002500"}}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["tokenizer = T5Tokenizer.from_pretrained('t5-small')\n","\n","# Prepare training datasets\n","train_dataset_full = KindleReviewDataset(tokenizer, train_df_full)\n","train_dataset_positive = KindleReviewDataset(tokenizer, train_df_positive)\n","train_dataset_negative = KindleReviewDataset(tokenizer, train_df_negative)\n","\n","# Prepare testing datasets\n","# These will be used later for evaluation\n","test_dataset_full = KindleReviewDataset(tokenizer, test_df_full)\n","test_dataset_positive = KindleReviewDataset(tokenizer, test_df_positive)\n","test_dataset_negative = KindleReviewDataset(tokenizer, test_df_negative)"],"metadata":{"id":"rzCf080b5yFh","executionInfo":{"status":"aborted","timestamp":1703285701971,"user_tz":300,"elapsed":4,"user":{"displayName":"Madelyn Dempsey","userId":"12178507243338002500"}}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["from transformers import Trainer, TrainingArguments\n","import torch\n","\n","print(df.columns)\n","print(len(df))\n","\n","# important to use google collab torch instead (if we want to pay, >>)\n","device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n","print(f\"Using device: {device}\")\n","\n","model = T5ForConditionalGeneration.from_pretrained('t5-small').to(device)\n","\n","# fine tuning a bit more aggressively\n","training_args = TrainingArguments(\n","    output_dir='./results',\n","    num_train_epochs=10,\n","    per_device_train_batch_size=4,\n","    warmup_steps=500,\n","    weight_decay=0.01,\n","    learning_rate=5e-5,  # Adjusted learning rate\n","    logging_dir='./logs',\n","    logging_steps=10,\n","    evaluation_strategy=\"no\",  # No evaluation dataset\n","    gradient_accumulation_steps=2,  # Adjust if needed\n","    seed=42  # Set a random seed\n",")\n"],"metadata":{"id":"wWX3f1MmGTOe","executionInfo":{"status":"aborted","timestamp":1703285701971,"user_tz":300,"elapsed":4,"user":{"displayName":"Madelyn Dempsey","userId":"12178507243338002500"}}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["def train_and_save_model(train_dataset, model_name):\n","    model = T5ForConditionalGeneration.from_pretrained('t5-small').to(device)\n","    trainer = Trainer(\n","        model=model,\n","        args=training_args,\n","        train_dataset=train_dataset,\n","    )\n","    trainer.train()\n","    model.save_pretrained(f\"./{model_name}\")\n","\n","# Train and save each model\n","train_and_save_model(train_dataset_full, \"model_full\")\n","train_and_save_model(train_dataset_positive, \"model_positive\")\n","train_and_save_model(train_dataset_negative, \"model_negative\")"],"metadata":{"id":"mnRffFef6Dcr","executionInfo":{"status":"aborted","timestamp":1703285701971,"user_tz":300,"elapsed":4,"user":{"displayName":"Madelyn Dempsey","userId":"12178507243338002500"}}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["import json\n","from transformers import pipeline\n","from datasets import load_metric\n","\n","# Function to summarize text\n","def summarize_text(text, tokenizer, model):\n","    inputs = tokenizer.encode(\"summarize: \" + text, return_tensors=\"pt\", max_length=512, truncation=True)\n","    inputs = inputs.to(model.device)\n","    outputs = model.generate(inputs, max_length=150, min_length=40, length_penalty=2.0, num_beams=4, early_stopping=True)\n","    return tokenizer.decode(outputs[0], skip_special_tokens=True)\n","\n","# Function to calculate ROUGE scores\n","def calculate_rouge(predictions, references):\n","    rouge = load_metric(\"rouge\")\n","    return rouge.compute(predictions=predictions, references=references)\n"],"metadata":{"id":"DUwigB9cGN7-","executionInfo":{"status":"aborted","timestamp":1703285701971,"user_tz":300,"elapsed":4,"user":{"displayName":"Madelyn Dempsey","userId":"12178507243338002500"}}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# Load trained models\n","model_full = T5ForConditionalGeneration.from_pretrained('./model_full').to(device)\n","model_positive = T5ForConditionalGeneration.from_pretrained('./model_positive').to(device)\n","model_negative = T5ForConditionalGeneration.from_pretrained('./model_negative').to(device)"],"metadata":{"id":"4RpG-toL9w9X","executionInfo":{"status":"aborted","timestamp":1703285701971,"user_tz":300,"elapsed":4,"user":{"displayName":"Madelyn Dempsey","userId":"12178507243338002500"}}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# # WARNING -> overwriting kindle test data with news test data\n","\n","# test_data = []\n","# with open('test.jsonl', 'r') as file:\n","#     for i, line in enumerate(file):\n","#         if i < 300:\n","#             test_data.append(json.loads(line))\n","#         else:\n","#             break\n","\n","# test_df_full = pd.DataFrame(test_data)\n","# test_df_full['sentiment'] = test_df_full['text'].apply(get_sentiment)\n","# test_df_full=test_df_full.rename(columns = {'text':'reviewText'})\n","\n","# # Prepare testing datasets\n","# test_df_positive = test_df_full[test_df_full['sentiment'] == 'POSITIVE'].drop(columns=['sentiment'])\n","# test_df_negative = test_df_full[test_df_full['sentiment'] == 'NEGATIVE'].drop(columns=['sentiment'])\n","\n","# # These will be used later for evaluation\n","# test_dataset_full = KindleReviewDataset(tokenizer, test_df_full)\n","# test_dataset_positive = KindleReviewDataset(tokenizer, test_df_positive)\n","# test_dataset_negative = KindleReviewDataset(tokenizer, test_df_negative)\n"],"metadata":{"id":"2k_5P48289e_","executionInfo":{"status":"aborted","timestamp":1703285701971,"user_tz":300,"elapsed":4,"user":{"displayName":"Madelyn Dempsey","userId":"12178507243338002500"}}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["test_df_full.head()"],"metadata":{"id":"Nvys-9MlAHGy","executionInfo":{"status":"aborted","timestamp":1703285701971,"user_tz":300,"elapsed":163484,"user":{"displayName":"Madelyn Dempsey","userId":"12178507243338002500"}}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["print(f'There are {len(test_df_positive)} positive rows, {len(test_df_negative)} negative rows, and {len(test_df_full)} total rows')"],"metadata":{"id":"qE-MLv6I-1Xc","executionInfo":{"status":"aborted","timestamp":1703285701971,"user_tz":300,"elapsed":163483,"user":{"displayName":"Madelyn Dempsey","userId":"12178507243338002500"}}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# Function to summarize a dataset using a model\n","def summarize_dataset(dataset, tokenizer, model): # CHANGES DEPENDING ON WHICH TEST SET\n","    return [summarize_text(text, tokenizer, model) for text in dataset.data['reviewText']]\n","\n","\n","# Summarize test data\n","summaries_full = {\n","    'full': summarize_dataset(test_dataset_full, tokenizer, model_full),\n","    'positive': summarize_dataset(test_dataset_full, tokenizer, model_positive),\n","    'negative': summarize_dataset(test_dataset_full, tokenizer, model_negative)\n","}\n","\n","print('done with full summaries')\n","\n","summaries_positive = {\n","    'full': summarize_dataset(test_dataset_positive, tokenizer, model_full),\n","    'positive': summarize_dataset(test_dataset_positive, tokenizer, model_positive),\n","    'negative': summarize_dataset(test_dataset_positive, tokenizer, model_negative)\n","}\n","\n","print('done with positive summaries')\n","\n","summaries_negative = {\n","    'full': summarize_dataset(test_dataset_negative, tokenizer, model_full),\n","    'positive': summarize_dataset(test_dataset_negative, tokenizer, model_positive),\n","    'negative': summarize_dataset(test_dataset_negative, tokenizer, model_negative)\n","}\n","\n","print('done with negative summaries')\n"],"metadata":{"id":"IEyBvR2eGyja","executionInfo":{"status":"aborted","timestamp":1703285701972,"user_tz":300,"elapsed":163482,"user":{"displayName":"Madelyn Dempsey","userId":"12178507243338002500"}}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# Function to analyze sentiment distribution\n","def analyze_sentiment_distribution(summaries):\n","    distribution = {}\n","    for model_type in summaries:\n","        positive_count = sum(1 for summary in summaries[model_type] if sentiment(summary)[0]['label'] == 'POSITIVE')\n","        total_count = len(summaries[model_type])\n","        distribution[model_type] = {\n","            'positive_percentage': (positive_count / total_count) * 100,\n","            'negative_percentage': 100 - (positive_count / total_count) * 100\n","        }\n","    return distribution\n","\n","# Analyze sentiment distribution\n","distribution_full = analyze_sentiment_distribution(summaries_full)\n","distribution_positive = analyze_sentiment_distribution(summaries_positive)\n","distribution_negative = analyze_sentiment_distribution(summaries_negative)\n"],"metadata":{"id":"c8VjWm4MGZ5s","executionInfo":{"status":"aborted","timestamp":1703285701972,"user_tz":300,"elapsed":163481,"user":{"displayName":"Madelyn Dempsey","userId":"12178507243338002500"}}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# Mapping number of words -> change in sentiment\n","\n","# adjusts so that 0 - 1.0 is negative; 1.0 - 2.0 is positive\n","def adjust_score(label, score):\n","  if label == 'POSITIVE':\n","    return score + 1.0\n","  else:\n","    return score\n","\n","# coordinates for overall plot (adding 3 diff datapoints for positive/negative/full)\n","x = []\n","y = []\n","\n","# coordinates for average plot (averaging the 3 datapoints)\n","avg_x = []\n","avg_y = []\n","\n","def word_to_score(data, summaries):\n","  global x, y, avg_x, avg_y\n","  for i, row in enumerate(data.iterrows()):\n","        original_review = row[1]['reviewText']\n","        num_words = len(original_review.split(' '))\n","\n","        original_sentiment = sentiment(row[1]['summary'])[0]\n","        original_label, original_score = original_sentiment['label'], original_sentiment['score']\n","\n","        count = 0\n","        for model_type in summaries: # positive, negative, and full\n","          generated_summary = summaries[model_type][i]\n","          generated_sentiment = sentiment(generated_summary)[0]\n","          generated_label, generated_score = generated_sentiment['label'], generated_sentiment['score']\n","\n","          x.append(num_words)\n","          y.append(adjust_score(original_label, original_score)/adjust_score(generated_label, generated_score))\n","\n","          count += adjust_score(generated_label, generated_score)\n","\n","        avg_x.append(num_words)\n","        avg_y.append(count / 3)\n","\n","word_to_score(test_df_positive, summaries_positive)\n","word_to_score(test_df_negative, summaries_negative)\n","word_to_score(test_df_full, summaries_full)\n"],"metadata":{"id":"GS-p2-NCwc0f","executionInfo":{"status":"aborted","timestamp":1703285701972,"user_tz":300,"elapsed":163480,"user":{"displayName":"Madelyn Dempsey","userId":"12178507243338002500"}}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["import matplotlib.pyplot as plt\n","\n","# plotting average\n","\n","plt.scatter(avg_x, avg_y)\n","\n","plt.xlabel('Number of Words in Text')\n","plt.ylabel('Average Change in Sentiment')\n","plt.title('Averaged Plot')\n","\n","plt.xlim((0,1500)) # did this to cut out outliers on x axis\n","\n","\n","plt.show()"],"metadata":{"id":"FEUCHqPi5UD_","executionInfo":{"status":"aborted","timestamp":1703285701972,"user_tz":300,"elapsed":163479,"user":{"displayName":"Madelyn Dempsey","userId":"12178507243338002500"}}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# plotting normal\n","\n","plt.scatter(x, y)\n","\n","plt.xlabel('Number of Words in Text')\n","plt.ylabel('Change in Sentiment')\n","plt.title('Plot')\n","\n","plt.xlim((0,1500)) # did this to cut out outliers on x axis\n","\n","plt.show()"],"metadata":{"id":"DJx_ZiVQ7arE","executionInfo":{"status":"aborted","timestamp":1703285701972,"user_tz":300,"elapsed":163477,"user":{"displayName":"Madelyn Dempsey","userId":"12178507243338002500"}}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["print(summaries_full)"],"metadata":{"id":"ob7QUdH4iAHu","executionInfo":{"status":"aborted","timestamp":1703285702242,"user_tz":300,"elapsed":2,"user":{"displayName":"Madelyn Dempsey","userId":"12178507243338002500"}}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["test_df_positive.head()"],"metadata":{"id":"n2FX4IU-Ph8G","executionInfo":{"status":"aborted","timestamp":1703285702242,"user_tz":300,"elapsed":2,"user":{"displayName":"Madelyn Dempsey","userId":"12178507243338002500"}}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["output_file = \"positive_data_output.txt\"\n","# train_df_full, test_df_full = split_data(df_full, r)\n","# train_df_positive, test_df_positive = split_data(df_positive, r)\n","# train_df_negative, test_df_negative = split_data(df_negative, r)\n","# Open the file in write mode\n","with open(output_file, \"w\") as file:\n","    file.write(f\"POSITIVE DATA:\\n\")\n","    file.write(\"\\n------------------------------------------------------\\n\\n\")\n","\n","    for i, row in enumerate(test_df_positive.iterrows()):\n","        original_review = row[1]['reviewText']\n","        original_summary = row[1]['summary']\n","\n","        positive_summary = summaries_positive['positive'][i]\n","        negative_summary = summaries_positive['negative'][i]\n","        full_summary = summaries_positive['full'][i]\n","\n","        # Write all information to file\n","        file.write(f\"Review: \\n{original_review}\\n\\n\")\n","        file.write(f\"Original Summary: \\n{original_summary}\\n\\n\")\n","        file.write(f\"Positive Model Summary: \\n{positive_summary}\\n\\n\")\n","        file.write(f\"Negative Model Summary: \\n{negative_summary}\\n\\n\")\n","        file.write(f\"Full Model Summary: \\n{full_summary}\\n\\n\")\n","        file.write(\"\\n------------------------------------------------------\\n\\n\")\n","\n","output_file = \"negative_data_output.txt\"\n","with open(output_file, \"w\") as file:\n","    file.write(f\"NEGATIVE DATA:\\n\")\n","    file.write(\"\\n------------------------------------------------------\\n\\n\")\n","\n","    for i, row in enumerate(test_df_negative.iterrows()):\n","        original_review = row[1]['reviewText']\n","        original_summary = row[1]['summary']\n","\n","        positive_summary = summaries_negative['positive'][i]\n","        negative_summary = summaries_negative['negative'][i]\n","        full_summary = summaries_negative['full'][i]\n","\n","        # Write all information to file\n","        file.write(f\"Review: \\n{original_review}\\n\\n\")\n","        file.write(f\"Original Summary: \\n{original_summary}\\n\\n\")\n","        file.write(f\"Positive Model Summary: \\n{positive_summary}\\n\\n\")\n","        file.write(f\"Negative Model Summary: \\n{negative_summary}\\n\\n\")\n","        file.write(f\"Full Model Summary: \\n{full_summary}\\n\\n\")\n","        file.write(\"\\n------------------------------------------------------\\n\\n\")\n","\n","output_file = \"full_data_output.txt\"\n","with open(output_file, \"w\") as file:\n","    file.write(f\"FULL DATA:\\n\")\n","    file.write(\"\\n------------------------------------------------------\\n\\n\")\n","\n","    for i, row in enumerate(test_df_full.iterrows()):\n","        original_review = row[1]['reviewText']\n","        original_summary = row[1]['summary']\n","\n","        positive_summary = summaries_full['positive'][i]\n","        negative_summary = summaries_full['negative'][i]\n","        full_summary = summaries_full['full'][i]\n","\n","        # Write all information to file\n","        file.write(f\"Review: \\n{original_review}\\n\\n\")\n","        file.write(f\"Original Summary: \\n{original_summary}\\n\\n\")\n","        file.write(f\"Positive Model Summary: \\n{positive_summary}\\n\\n\")\n","        file.write(f\"Negative Model Summary: \\n{negative_summary}\\n\\n\")\n","        file.write(f\"Full Model Summary: \\n{full_summary}\\n\\n\")\n","        file.write(\"\\n------------------------------------------------------\\n\\n\")\n","\n"],"metadata":{"id":"5IGjIBDJAE5A","executionInfo":{"status":"aborted","timestamp":1703285702242,"user_tz":300,"elapsed":2,"user":{"displayName":"Madelyn Dempsey","userId":"12178507243338002500"}}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# Display results\n","def display_results(distribution, test_type):\n","    print(f\"Results for {test_type} Test Data:\")\n","    for model_type, dist in distribution.items():\n","        print(f\"  Summaries by {model_type} Model - Positive: {dist['positive_percentage']:.2f}%, Negative: {dist['negative_percentage']:.2f}%\")\n","    print()\n","\n","display_results(distribution_full, \"Full\")\n","display_results(distribution_positive, \"Positive\")\n","display_results(distribution_negative, \"Negative\")\n"],"metadata":{"id":"q3_PmgIyHMp4","executionInfo":{"status":"aborted","timestamp":1703285702242,"user_tz":300,"elapsed":2,"user":{"displayName":"Madelyn Dempsey","userId":"12178507243338002500"}}},"execution_count":null,"outputs":[]}]}