usnews_preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Import modules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.simplefilter(action='ignore', category=FutureWarning)\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import re\n",
    "import nltk\n",
    "import datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_full(x):\n",
    "    pd.set_option('display.max_rows', len(x))\n",
    "    pd.set_option('display.max_columns', None)\n",
    "    pd.set_option('display.width', 2000)\n",
    "    pd.set_option('display.float_format', '{:20,.2f}'.format)\n",
    "    pd.set_option('display.max_colwidth', None)\n",
    "    display(x)\n",
    "    pd.set_option('display.max_row', 50)\n",
    "    pd.set_option('display.max_column', 100)\n",
    "    pd.reset_option('display.width')\n",
    "    pd.reset_option('display.float_format')\n",
    "    pd.reset_option('display.max_colwidth')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Title</th>\n",
       "      <th>Description</th>\n",
       "      <th>Content</th>\n",
       "      <th>URL</th>\n",
       "      <th>Date</th>\n",
       "      <th>Author</th>\n",
       "      <th>Category</th>\n",
       "      <th>NewsProvider</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>7813</th>\n",
       "      <td>Growing role for AI to combat space threats</td>\n",
       "      <td>Retired Air Force Lt. Gen. Chris Bogdan sees t...</td>\n",
       "      <td>If Russia or China were to launch a missile at...</td>\n",
       "      <td>https://www.politico.com/news/2020/01/10/artif...</td>\n",
       "      <td>2020-01-10</td>\n",
       "      <td>Jacqueline Feldscher</td>\n",
       "      <td>Space</td>\n",
       "      <td>Politico</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            Title  \\\n",
       "7813  Growing role for AI to combat space threats   \n",
       "\n",
       "                                            Description  \\\n",
       "7813  Retired Air Force Lt. Gen. Chris Bogdan sees t...   \n",
       "\n",
       "                                                Content  \\\n",
       "7813  If Russia or China were to launch a missile at...   \n",
       "\n",
       "                                                    URL        Date  \\\n",
       "7813  https://www.politico.com/news/2020/01/10/artif...  2020-01-10   \n",
       "\n",
       "                    Author Category NewsProvider  \n",
       "7813  Jacqueline Feldscher    Space     Politico  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "Title           7814\n",
       "Description     7814\n",
       "Content         7814\n",
       "URL             7814\n",
       "Date            7814\n",
       "Author          7814\n",
       "Category        7555\n",
       "NewsProvider    7814\n",
       "dtype: int64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Title</th>\n",
       "      <th>Description</th>\n",
       "      <th>Content</th>\n",
       "      <th>URL</th>\n",
       "      <th>Date</th>\n",
       "      <th>Author</th>\n",
       "      <th>Category</th>\n",
       "      <th>NewsProvider</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Why It Took So Long For Politicians To Treat T...</td>\n",
       "      <td>America is a little matryoshka doll of panic r...</td>\n",
       "      <td>America is a little doll of panic right now; p...</td>\n",
       "      <td>https://fivethirtyeight.com/features/why-it-to...</td>\n",
       "      <td>2020-07-16T13:00:27+00:00</td>\n",
       "      <td>Clare Malone</td>\n",
       "      <td>Coronavirus</td>\n",
       "      <td>FiveThirtyEight</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               Title  \\\n",
       "0  Why It Took So Long For Politicians To Treat T...   \n",
       "\n",
       "                                         Description  \\\n",
       "0  America is a little matryoshka doll of panic r...   \n",
       "\n",
       "                                             Content  \\\n",
       "0  America is a little doll of panic right now; p...   \n",
       "\n",
       "                                                 URL  \\\n",
       "0  https://fivethirtyeight.com/features/why-it-to...   \n",
       "\n",
       "                        Date        Author     Category     NewsProvider  \n",
       "0  2020-07-16T13:00:27+00:00  Clare Malone  Coronavirus  FiveThirtyEight  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "Title           502\n",
       "Description     502\n",
       "Content         502\n",
       "URL             502\n",
       "Date            502\n",
       "Author          476\n",
       "Category        501\n",
       "NewsProvider    502\n",
       "dtype: int64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Title</th>\n",
       "      <th>Description</th>\n",
       "      <th>Content</th>\n",
       "      <th>URL</th>\n",
       "      <th>Date</th>\n",
       "      <th>Author</th>\n",
       "      <th>Category</th>\n",
       "      <th>NewsProvider</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3719</th>\n",
       "      <td>Trump says Wisconsin governor to allow federal...</td>\n",
       "      <td>The governor already activated the Wisconsin N...</td>\n",
       "      <td>President Trump tweeted on Wednesday that Wisc...</td>\n",
       "      <td>https://www.axios.com/jacob-blake-wisconsin-ke...</td>\n",
       "      <td>2020-08-26T18:55:48.417161Z</td>\n",
       "      <td>Ursula Perano</td>\n",
       "      <td>Politics &amp; Policy</td>\n",
       "      <td>Axios</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  Title  \\\n",
       "3719  Trump says Wisconsin governor to allow federal...   \n",
       "\n",
       "                                            Description  \\\n",
       "3719  The governor already activated the Wisconsin N...   \n",
       "\n",
       "                                                Content  \\\n",
       "3719  President Trump tweeted on Wednesday that Wisc...   \n",
       "\n",
       "                                                    URL  \\\n",
       "3719  https://www.axios.com/jacob-blake-wisconsin-ke...   \n",
       "\n",
       "                             Date         Author           Category  \\\n",
       "3719  2020-08-26T18:55:48.417161Z  Ursula Perano  Politics & Policy   \n",
       "\n",
       "     NewsProvider  \n",
       "3719        Axios  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "Title           3720\n",
       "Description     3720\n",
       "Content         3720\n",
       "URL             3720\n",
       "Date            3720\n",
       "Author          3720\n",
       "Category        3720\n",
       "NewsProvider    3720\n",
       "dtype: int64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Title</th>\n",
       "      <th>Description</th>\n",
       "      <th>Content</th>\n",
       "      <th>URL</th>\n",
       "      <th>Date</th>\n",
       "      <th>Author</th>\n",
       "      <th>Category</th>\n",
       "      <th>NewsProvider</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>20389</th>\n",
       "      <td>22 Years Later: How China Destroyed Legacy of ...</td>\n",
       "      <td>2019 was the year that the People’s Republic o...</td>\n",
       "      <td>2019 was the year that the People’s Republic o...</td>\n",
       "      <td>https://www.breitbart.com/europe/2020/01/01/22...</td>\n",
       "      <td>2020-01-01T11:20:30+00:00</td>\n",
       "      <td>Kurt Zindulka</td>\n",
       "      <td>Politics,London / Europe,Asia</td>\n",
       "      <td>Breitbart</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   Title  \\\n",
       "20389  22 Years Later: How China Destroyed Legacy of ...   \n",
       "\n",
       "                                             Description  \\\n",
       "20389  2019 was the year that the People’s Republic o...   \n",
       "\n",
       "                                                 Content  \\\n",
       "20389  2019 was the year that the People’s Republic o...   \n",
       "\n",
       "                                                     URL  \\\n",
       "20389  https://www.breitbart.com/europe/2020/01/01/22...   \n",
       "\n",
       "                            Date         Author  \\\n",
       "20389  2020-01-01T11:20:30+00:00  Kurt Zindulka   \n",
       "\n",
       "                            Category NewsProvider  \n",
       "20389  Politics,London / Europe,Asia    Breitbart  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "Title           20214\n",
       "Description     20214\n",
       "Content         20214\n",
       "URL             20214\n",
       "Date            20214\n",
       "Author          20214\n",
       "Category        20214\n",
       "NewsProvider    20214\n",
       "dtype: int64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Load politico data set\n",
    "politico = pd.read_csv(\"data/politicopolitics.csv\")\n",
    "\n",
    "# Add column that indicates news provider\n",
    "politico[\"NewsProvider\"] = \"Politico\"\n",
    "\n",
    "# Drop articles that have no title\n",
    "politico = politico[~politico[\"Title\"].isnull()]\n",
    "\n",
    "# Drop articles that have no content\n",
    "politico = politico[~politico[\"Content\"].isnull()]\n",
    "\n",
    "# Drop articles with the same header\n",
    "politico = politico.drop_duplicates(subset='Title', keep=\"first\")\n",
    "\n",
    "# Create new year, day, and month columns\n",
    "politico['Date'] = politico.apply(lambda x: x[\"Date\"][:10], axis = \"columns\")\n",
    "politico['Year'] = politico.apply(lambda x: x[\"Date\"][6:], axis = \"columns\")\n",
    "politico['Day'] = politico.apply(lambda x: x[\"Date\"][3:5], axis = \"columns\")\n",
    "politico['Month'] = politico.apply(lambda x: x[\"Date\"][:2], axis = \"columns\")\n",
    "\n",
    "# Create clean date column\n",
    "politico['Date'] = politico.apply(lambda x: x[\"Year\"] + \"-\" + x[\"Month\"] + \"-\" + x[\"Day\"], axis = \"columns\")\n",
    "\n",
    "# Only keep articles from 2020\n",
    "politico = politico[politico[\"Year\"] == \"2020\"].reset_index(drop = True)\n",
    "\n",
    "del politico[\"Year\"]\n",
    "del politico[\"Day\"]\n",
    "del politico[\"Month\"]\n",
    "\n",
    "# Only capitalize first letter of author string\n",
    "politico['Author'] = politico.apply(lambda x: str(x[\"Author\"]).title(), axis = \"columns\")\n",
    "\n",
    "# Quick overview\n",
    "display(politico.tail(1))\n",
    "display(politico.count())\n",
    "\n",
    "# Load fivethirtyeight\n",
    "fivethirty = pd.read_csv(\"data/fivethiryeightpolitics.csv\")\n",
    "\n",
    "# Add column that indicates news provider\n",
    "fivethirty[\"NewsProvider\"] = \"FiveThirtyEight\" \n",
    "\n",
    "# Drop articles with the same header\n",
    "fivethirty = fivethirty.drop_duplicates(subset='Title', keep=\"first\")\n",
    "\n",
    "# Drop articles that have no content\n",
    "fivethirty = fivethirty[~fivethirty[\"Content\"].isnull()]\n",
    "\n",
    "# Quick overview\n",
    "display(fivethirty.head(1))\n",
    "display(fivethirty.count())\n",
    "\n",
    "# Load axios\n",
    "axios = pd.read_csv(\"data/axiospolitics.csv\")\n",
    "\n",
    "# Add column that indicates news provider\n",
    "axios[\"NewsProvider\"] = \"Axios\" \n",
    "\n",
    "# Quick overview\n",
    "display(axios.tail(1))\n",
    "display(axios.count())\n",
    "\n",
    "# Load Breitbart\n",
    "breitbart = pd.read_csv(\"data/breitbartpolitics.csv\")\n",
    "\n",
    "# Add column that indicates news provider\n",
    "breitbart[\"NewsProvider\"] = \"Breitbart\"\n",
    "\n",
    "# Drop articles with the same header\n",
    "breitbart = breitbart.drop_duplicates(subset='Title', keep=\"first\")\n",
    "\n",
    "# Drop articles that have no content\n",
    "breitbart = breitbart[~breitbart[\"Content\"].isnull()]\n",
    "\n",
    "# Drop articles that have no date\n",
    "breitbart = breitbart[~breitbart[\"Date\"].isnull()]\n",
    "\n",
    "# Quick overview\n",
    "display(breitbart.tail(1))\n",
    "display(breitbart.count())\n",
    "\n",
    "# Merge as one object\n",
    "usnews = axios.append([breitbart, fivethirty, politico])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data wrangling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Title</th>\n",
       "      <th>Description</th>\n",
       "      <th>Content</th>\n",
       "      <th>URL</th>\n",
       "      <th>Date</th>\n",
       "      <th>Author</th>\n",
       "      <th>Category</th>\n",
       "      <th>NewsProvider</th>\n",
       "      <th>Year</th>\n",
       "      <th>Day</th>\n",
       "      <th>Month</th>\n",
       "      <th>CurrentDate</th>\n",
       "      <th>Datetime</th>\n",
       "      <th>Week</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>What we're watching in 2020</td>\n",
       "      <td>Trump's post-impeachment future, Big Tech regu...</td>\n",
       "      <td>This is going to be a momentous presidential e...</td>\n",
       "      <td>https://www.axios.com/what-we-are-watching-202...</td>\n",
       "      <td>2020-01-01</td>\n",
       "      <td>Jim VandeHei,Mike Allen</td>\n",
       "      <td>Politics &amp; Policy</td>\n",
       "      <td>Axios</td>\n",
       "      <td>2020</td>\n",
       "      <td>01</td>\n",
       "      <td>01</td>\n",
       "      <td>2020-08-27</td>\n",
       "      <td>2020-01-01</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Chief Justice Roberts says Americans may \"take...</td>\n",
       "      <td>It reads as a mission statement ahead of Presi...</td>\n",
       "      <td>Supreme Court Chief Justice John Roberts warne...</td>\n",
       "      <td>https://www.axios.com/chief-justice-john-rober...</td>\n",
       "      <td>2020-01-01</td>\n",
       "      <td>Jacob Knutson</td>\n",
       "      <td>Politics &amp; Policy</td>\n",
       "      <td>Axios</td>\n",
       "      <td>2020</td>\n",
       "      <td>01</td>\n",
       "      <td>01</td>\n",
       "      <td>2020-08-27</td>\n",
       "      <td>2020-01-01</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Trump's twin war threats</td>\n",
       "      <td>They will truly test his diplomatic mix of blu...</td>\n",
       "      <td>President Trump suddenly faces two global cris...</td>\n",
       "      <td>https://www.axios.com/north-korea-iran-trump-w...</td>\n",
       "      <td>2020-01-01</td>\n",
       "      <td>Mike Allen,Jim VandeHei</td>\n",
       "      <td>Politics &amp; Policy</td>\n",
       "      <td>Axios</td>\n",
       "      <td>2020</td>\n",
       "      <td>01</td>\n",
       "      <td>01</td>\n",
       "      <td>2020-08-27</td>\n",
       "      <td>2020-01-01</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Pollak: Welcome to 2020, the Year of the Good ...</td>\n",
       "      <td>The year 2020 begins with incredible potential...</td>\n",
       "      <td>The year 2020 begins with incredible potential...</td>\n",
       "      <td>https://www.breitbart.com/2020-election/2019/1...</td>\n",
       "      <td>2020-01-01</td>\n",
       "      <td>Joel B. Pollak</td>\n",
       "      <td>Politics,Media,2020 Election</td>\n",
       "      <td>Breitbart</td>\n",
       "      <td>2020</td>\n",
       "      <td>01</td>\n",
       "      <td>01</td>\n",
       "      <td>2020-08-27</td>\n",
       "      <td>2020-01-01</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Warren Blames Unrest on Trump's 'Recklessness'...</td>\n",
       "      <td>Sen. Elizabeth Warren blamed President Trump f...</td>\n",
       "      <td>Sen. Elizabeth Warren (D-MA) on Tuesday blamed...</td>\n",
       "      <td>https://www.breitbart.com/middle-east/2019/12/...</td>\n",
       "      <td>2020-01-01</td>\n",
       "      <td>Hannah Bleau</td>\n",
       "      <td>Politics,National Security,Israel / Middle East</td>\n",
       "      <td>Breitbart</td>\n",
       "      <td>2020</td>\n",
       "      <td>01</td>\n",
       "      <td>01</td>\n",
       "      <td>2020-08-27</td>\n",
       "      <td>2020-01-01</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32434</th>\n",
       "      <td>A Well-Connected GOP Strategist Is Helping Kan...</td>\n",
       "      <td>Lane Ruhland filed West’s filing paperwork and...</td>\n",
       "      <td>Kanye West has officially submitted signature...</td>\n",
       "      <td>https://www.vice.com/en_us/article/akzy3b/a-we...</td>\n",
       "      <td>2020-08-5</td>\n",
       "      <td>Cameron Joseph</td>\n",
       "      <td>culture, news, lgbtq, politics, journalism, vi...</td>\n",
       "      <td>Vice</td>\n",
       "      <td>2020</td>\n",
       "      <td>5</td>\n",
       "      <td>08</td>\n",
       "      <td>2020-08-27</td>\n",
       "      <td>2020-08-05</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32435</th>\n",
       "      <td>Kanye West Has a Senior GOP Strategist Helping...</td>\n",
       "      <td>“Would you help me get Kanye West on the ballo...</td>\n",
       "      <td>A top Colorado GOP strategist is helping Kanye...</td>\n",
       "      <td>https://www.vice.com/en_us/article/889kv5/kany...</td>\n",
       "      <td>2020-08-5</td>\n",
       "      <td>Cameron Joseph</td>\n",
       "      <td>culture, news, lgbtq, politics, journalism, vi...</td>\n",
       "      <td>Vice</td>\n",
       "      <td>2020</td>\n",
       "      <td>5</td>\n",
       "      <td>08</td>\n",
       "      <td>2020-08-27</td>\n",
       "      <td>2020-08-05</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32436</th>\n",
       "      <td>A Malaysian Minister Was Caught Vaping in Parl...</td>\n",
       "      <td>Malaysia’s Foreign Affairs Minister tried to s...</td>\n",
       "      <td>A Malaysian politician has apologized for vapi...</td>\n",
       "      <td>https://www.vice.com/en_us/article/bv8pad/mala...</td>\n",
       "      <td>2020-08-7</td>\n",
       "      <td>Heather Chen</td>\n",
       "      <td>culture, news, lgbtq, politics, journalism, vi...</td>\n",
       "      <td>Vice</td>\n",
       "      <td>2020</td>\n",
       "      <td>7</td>\n",
       "      <td>08</td>\n",
       "      <td>2020-08-27</td>\n",
       "      <td>2020-08-07</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32437</th>\n",
       "      <td>Hackers Hijack Reddit Mod Accounts to Post Pro...</td>\n",
       "      <td>Hackers took control and defaced several large...</td>\n",
       "      <td>Hackers took control of more than a dozen subr...</td>\n",
       "      <td>https://www.vice.com/en_us/article/y3zx7g/hack...</td>\n",
       "      <td>2020-08-7</td>\n",
       "      <td>Lorenzo Franceschi-Bicchierai</td>\n",
       "      <td>culture, news, lgbtq, politics, journalism, vi...</td>\n",
       "      <td>Vice</td>\n",
       "      <td>2020</td>\n",
       "      <td>7</td>\n",
       "      <td>08</td>\n",
       "      <td>2020-08-27</td>\n",
       "      <td>2020-08-07</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32438</th>\n",
       "      <td>Charges Against 24 High-Profile Activists Part...</td>\n",
       "      <td>Human Rights Watch China director Sophie Richa...</td>\n",
       "      <td>Hong Kong handed down charges against two doze...</td>\n",
       "      <td>https://www.vice.com/en_us/article/7kpdaq/char...</td>\n",
       "      <td>2020-08-7</td>\n",
       "      <td>Sammy Westfall</td>\n",
       "      <td>culture, news, lgbtq, politics, journalism, vi...</td>\n",
       "      <td>Vice</td>\n",
       "      <td>2020</td>\n",
       "      <td>7</td>\n",
       "      <td>08</td>\n",
       "      <td>2020-08-27</td>\n",
       "      <td>2020-08-07</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>32439 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   Title  \\\n",
       "0                            What we're watching in 2020   \n",
       "1      Chief Justice Roberts says Americans may \"take...   \n",
       "2                               Trump's twin war threats   \n",
       "3      Pollak: Welcome to 2020, the Year of the Good ...   \n",
       "4      Warren Blames Unrest on Trump's 'Recklessness'...   \n",
       "...                                                  ...   \n",
       "32434  A Well-Connected GOP Strategist Is Helping Kan...   \n",
       "32435  Kanye West Has a Senior GOP Strategist Helping...   \n",
       "32436  A Malaysian Minister Was Caught Vaping in Parl...   \n",
       "32437  Hackers Hijack Reddit Mod Accounts to Post Pro...   \n",
       "32438  Charges Against 24 High-Profile Activists Part...   \n",
       "\n",
       "                                             Description  \\\n",
       "0      Trump's post-impeachment future, Big Tech regu...   \n",
       "1      It reads as a mission statement ahead of Presi...   \n",
       "2      They will truly test his diplomatic mix of blu...   \n",
       "3      The year 2020 begins with incredible potential...   \n",
       "4      Sen. Elizabeth Warren blamed President Trump f...   \n",
       "...                                                  ...   \n",
       "32434  Lane Ruhland filed West’s filing paperwork and...   \n",
       "32435  “Would you help me get Kanye West on the ballo...   \n",
       "32436  Malaysia’s Foreign Affairs Minister tried to s...   \n",
       "32437  Hackers took control and defaced several large...   \n",
       "32438  Human Rights Watch China director Sophie Richa...   \n",
       "\n",
       "                                                 Content  \\\n",
       "0      This is going to be a momentous presidential e...   \n",
       "1      Supreme Court Chief Justice John Roberts warne...   \n",
       "2      President Trump suddenly faces two global cris...   \n",
       "3      The year 2020 begins with incredible potential...   \n",
       "4      Sen. Elizabeth Warren (D-MA) on Tuesday blamed...   \n",
       "...                                                  ...   \n",
       "32434   Kanye West has officially submitted signature...   \n",
       "32435  A top Colorado GOP strategist is helping Kanye...   \n",
       "32436  A Malaysian politician has apologized for vapi...   \n",
       "32437  Hackers took control of more than a dozen subr...   \n",
       "32438  Hong Kong handed down charges against two doze...   \n",
       "\n",
       "                                                     URL        Date  \\\n",
       "0      https://www.axios.com/what-we-are-watching-202...  2020-01-01   \n",
       "1      https://www.axios.com/chief-justice-john-rober...  2020-01-01   \n",
       "2      https://www.axios.com/north-korea-iran-trump-w...  2020-01-01   \n",
       "3      https://www.breitbart.com/2020-election/2019/1...  2020-01-01   \n",
       "4      https://www.breitbart.com/middle-east/2019/12/...  2020-01-01   \n",
       "...                                                  ...         ...   \n",
       "32434  https://www.vice.com/en_us/article/akzy3b/a-we...   2020-08-5   \n",
       "32435  https://www.vice.com/en_us/article/889kv5/kany...   2020-08-5   \n",
       "32436  https://www.vice.com/en_us/article/bv8pad/mala...   2020-08-7   \n",
       "32437  https://www.vice.com/en_us/article/y3zx7g/hack...   2020-08-7   \n",
       "32438  https://www.vice.com/en_us/article/7kpdaq/char...   2020-08-7   \n",
       "\n",
       "                              Author  \\\n",
       "0            Jim VandeHei,Mike Allen   \n",
       "1                      Jacob Knutson   \n",
       "2            Mike Allen,Jim VandeHei   \n",
       "3                     Joel B. Pollak   \n",
       "4                       Hannah Bleau   \n",
       "...                              ...   \n",
       "32434                 Cameron Joseph   \n",
       "32435                 Cameron Joseph   \n",
       "32436                   Heather Chen   \n",
       "32437  Lorenzo Franceschi-Bicchierai   \n",
       "32438                 Sammy Westfall   \n",
       "\n",
       "                                                Category NewsProvider  Year  \\\n",
       "0                                      Politics & Policy        Axios  2020   \n",
       "1                                      Politics & Policy        Axios  2020   \n",
       "2                                      Politics & Policy        Axios  2020   \n",
       "3                           Politics,Media,2020 Election    Breitbart  2020   \n",
       "4        Politics,National Security,Israel / Middle East    Breitbart  2020   \n",
       "...                                                  ...          ...   ...   \n",
       "32434  culture, news, lgbtq, politics, journalism, vi...         Vice  2020   \n",
       "32435  culture, news, lgbtq, politics, journalism, vi...         Vice  2020   \n",
       "32436  culture, news, lgbtq, politics, journalism, vi...         Vice  2020   \n",
       "32437  culture, news, lgbtq, politics, journalism, vi...         Vice  2020   \n",
       "32438  culture, news, lgbtq, politics, journalism, vi...         Vice  2020   \n",
       "\n",
       "      Day Month CurrentDate   Datetime  Week  \n",
       "0      01    01  2020-08-27 2020-01-01     1  \n",
       "1      01    01  2020-08-27 2020-01-01     1  \n",
       "2      01    01  2020-08-27 2020-01-01     1  \n",
       "3      01    01  2020-08-27 2020-01-01     1  \n",
       "4      01    01  2020-08-27 2020-01-01     1  \n",
       "...    ..   ...         ...        ...   ...  \n",
       "32434   5    08  2020-08-27 2020-08-05    32  \n",
       "32435   5    08  2020-08-27 2020-08-05    32  \n",
       "32436   7    08  2020-08-27 2020-08-07    33  \n",
       "32437   7    08  2020-08-27 2020-08-07    33  \n",
       "32438   7    08  2020-08-27 2020-08-07    33  \n",
       "\n",
       "[32439 rows x 14 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Transform to right data type\n",
    "column_names = [\"Title\", \"Description\", \"Content\", \"Date\"]\n",
    "for column_name in column_names:\n",
    "    usnews[column_name] = usnews.apply(lambda x: str(x[column_name]), axis = 1)\n",
    "    \n",
    "# Clean date column and sort by date\n",
    "usnews['Date'] = usnews.apply(lambda x: x[\"Date\"][:10], axis = \"columns\")\n",
    "usnews['Year'] = usnews.apply(lambda x: x[\"Date\"][:4], axis = \"columns\")\n",
    "usnews['Month'] = usnews.apply(lambda x: x[\"Date\"][5:7], axis = \"columns\")\n",
    "usnews['Day'] = usnews.apply(lambda x: x[\"Date\"][8:], axis = \"columns\")\n",
    "usnews['CurrentDate'] = datetime.datetime(2020, 8, 27)\n",
    "usnews['Datetime'] = usnews.apply(lambda x: pd.to_datetime(x[\"Date\"], format = '%Y-%m-%d'), axis = \"columns\")\n",
    "usnews['Week'] = usnews.apply(lambda x: 35 + int((x[\"Datetime\"] - x[\"CurrentDate\"]).total_seconds()/604800), axis = 1)\n",
    "usnews = usnews.sort_values(by=[\"Date\"])\n",
    "\n",
    "# Only keep articles from 2020\n",
    "usnews = usnews[usnews[\"Year\"] == \"2020\"].reset_index(drop = True)\n",
    "usnews"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Vader Sentiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer\n",
    "analyzer = SentimentIntensityAnalyzer()\n",
    "\n",
    "# Negative scores\n",
    "usnews[\"negSentiment\"] = usnews.apply(lambda x: analyzer.polarity_scores(x[\"Content\"])[\"neg\"], axis=\"columns\")\n",
    " \n",
    "# Neutral scores\n",
    "usnews[\"neuSentiment\"] = usnews.apply(lambda x: analyzer.polarity_scores(x[\"Content\"])[\"neu\"], axis=\"columns\")\n",
    " \n",
    "# Positive scores\n",
    "usnews[\"posSentiment\"] = usnews.apply(lambda x: analyzer.polarity_scores(x[\"Content\"])[\"pos\"], axis=\"columns\")\n",
    "\n",
    "# Compound scores\n",
    "usnews[\"compoundSentiment\"] = usnews.apply(lambda x: analyzer.polarity_scores(x[\"Content\"])[\"compound\"], axis=\"columns\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Textblob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from textblob import TextBlob\n",
    "usnews[\"polarity\"] = usnews['Content'].apply(lambda x: TextBlob(x).sentiment[0])\n",
    "usnews[\"subjectivity\"] = usnews['Content'].apply(lambda x: TextBlob(x).sentiment[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Text preparation for LDA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Title</th>\n",
       "      <th>Description</th>\n",
       "      <th>Content</th>\n",
       "      <th>URL</th>\n",
       "      <th>Date</th>\n",
       "      <th>Author</th>\n",
       "      <th>Category</th>\n",
       "      <th>NewsProvider</th>\n",
       "      <th>Year</th>\n",
       "      <th>Day</th>\n",
       "      <th>...</th>\n",
       "      <th>neuSentiment</th>\n",
       "      <th>posSentiment</th>\n",
       "      <th>compoundSentiment</th>\n",
       "      <th>polarity</th>\n",
       "      <th>subjectivity</th>\n",
       "      <th>Content_clean</th>\n",
       "      <th>Content_words</th>\n",
       "      <th>Content_filteredwords</th>\n",
       "      <th>Content_stemmedwords</th>\n",
       "      <th>Totalwords</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>What we're watching in 2020</td>\n",
       "      <td>Trump's post-impeachment future, Big Tech regu...</td>\n",
       "      <td>This is going to be a momentous presidential e...</td>\n",
       "      <td>https://www.axios.com/what-we-are-watching-202...</td>\n",
       "      <td>2020-01-01</td>\n",
       "      <td>Jim VandeHei,Mike Allen</td>\n",
       "      <td>Politics &amp; Policy</td>\n",
       "      <td>Axios</td>\n",
       "      <td>2020</td>\n",
       "      <td>01</td>\n",
       "      <td>...</td>\n",
       "      <td>0.858</td>\n",
       "      <td>0.091</td>\n",
       "      <td>0.9834</td>\n",
       "      <td>0.168711</td>\n",
       "      <td>0.358021</td>\n",
       "      <td>this is going to be a momentous presidential e...</td>\n",
       "      <td>[this, is, going, to, be, a, momentous, presid...</td>\n",
       "      <td>[going, momentous, presidential, election, yea...</td>\n",
       "      <td>[go, moment, presidenti, elect, year, face, re...</td>\n",
       "      <td>393</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Chief Justice Roberts says Americans may \"take...</td>\n",
       "      <td>It reads as a mission statement ahead of Presi...</td>\n",
       "      <td>Supreme Court Chief Justice John Roberts warne...</td>\n",
       "      <td>https://www.axios.com/chief-justice-john-rober...</td>\n",
       "      <td>2020-01-01</td>\n",
       "      <td>Jacob Knutson</td>\n",
       "      <td>Politics &amp; Policy</td>\n",
       "      <td>Axios</td>\n",
       "      <td>2020</td>\n",
       "      <td>01</td>\n",
       "      <td>...</td>\n",
       "      <td>0.796</td>\n",
       "      <td>0.149</td>\n",
       "      <td>0.9779</td>\n",
       "      <td>0.144646</td>\n",
       "      <td>0.355859</td>\n",
       "      <td>supreme court chief justice john roberts warne...</td>\n",
       "      <td>[supreme, court, chief, justice, john, roberts...</td>\n",
       "      <td>[supreme, court, chief, justice, john, roberts...</td>\n",
       "      <td>[suprem, court, chief, justic, john, robert, w...</td>\n",
       "      <td>145</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Trump's twin war threats</td>\n",
       "      <td>They will truly test his diplomatic mix of blu...</td>\n",
       "      <td>President Trump suddenly faces two global cris...</td>\n",
       "      <td>https://www.axios.com/north-korea-iran-trump-w...</td>\n",
       "      <td>2020-01-01</td>\n",
       "      <td>Mike Allen,Jim VandeHei</td>\n",
       "      <td>Politics &amp; Policy</td>\n",
       "      <td>Axios</td>\n",
       "      <td>2020</td>\n",
       "      <td>01</td>\n",
       "      <td>...</td>\n",
       "      <td>0.840</td>\n",
       "      <td>0.040</td>\n",
       "      <td>-0.9804</td>\n",
       "      <td>0.029773</td>\n",
       "      <td>0.298242</td>\n",
       "      <td>president trump suddenly faces two global cris...</td>\n",
       "      <td>[president, trump, suddenly, faces, two, globa...</td>\n",
       "      <td>[president, trump, suddenly, faces, two, globa...</td>\n",
       "      <td>[presid, trump, suddenli, face, two, global, c...</td>\n",
       "      <td>169</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Pollak: Welcome to 2020, the Year of the Good ...</td>\n",
       "      <td>The year 2020 begins with incredible potential...</td>\n",
       "      <td>The year 2020 begins with incredible potential...</td>\n",
       "      <td>https://www.breitbart.com/2020-election/2019/1...</td>\n",
       "      <td>2020-01-01</td>\n",
       "      <td>Joel B. Pollak</td>\n",
       "      <td>Politics,Media,2020 Election</td>\n",
       "      <td>Breitbart</td>\n",
       "      <td>2020</td>\n",
       "      <td>01</td>\n",
       "      <td>...</td>\n",
       "      <td>0.760</td>\n",
       "      <td>0.110</td>\n",
       "      <td>-0.9320</td>\n",
       "      <td>0.052025</td>\n",
       "      <td>0.445746</td>\n",
       "      <td>the year 2020 begins with incredible potential...</td>\n",
       "      <td>[the, year, 2020, begins, with, incredible, po...</td>\n",
       "      <td>[year, 2020, begins, incredible, potential, su...</td>\n",
       "      <td>[year, 2020, begin, incred, potenti, support, ...</td>\n",
       "      <td>240</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Warren Blames Unrest on Trump's 'Recklessness'...</td>\n",
       "      <td>Sen. Elizabeth Warren blamed President Trump f...</td>\n",
       "      <td>Sen. Elizabeth Warren (D-MA) on Tuesday blamed...</td>\n",
       "      <td>https://www.breitbart.com/middle-east/2019/12/...</td>\n",
       "      <td>2020-01-01</td>\n",
       "      <td>Hannah Bleau</td>\n",
       "      <td>Politics,National Security,Israel / Middle East</td>\n",
       "      <td>Breitbart</td>\n",
       "      <td>2020</td>\n",
       "      <td>01</td>\n",
       "      <td>...</td>\n",
       "      <td>0.800</td>\n",
       "      <td>0.056</td>\n",
       "      <td>-0.9968</td>\n",
       "      <td>-0.020271</td>\n",
       "      <td>0.459366</td>\n",
       "      <td>sen. elizabeth warren (d-ma) on tuesday blamed...</td>\n",
       "      <td>[sen, elizabeth, warren, d-ma, on, tuesday, bl...</td>\n",
       "      <td>[sen, elizabeth, warren, d-ma, tuesday, blamed...</td>\n",
       "      <td>[sen, elizabeth, warren, d-ma, tuesday, blame,...</td>\n",
       "      <td>390</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32434</th>\n",
       "      <td>A Well-Connected GOP Strategist Is Helping Kan...</td>\n",
       "      <td>Lane Ruhland filed West’s filing paperwork and...</td>\n",
       "      <td>Kanye West has officially submitted signature...</td>\n",
       "      <td>https://www.vice.com/en_us/article/akzy3b/a-we...</td>\n",
       "      <td>2020-08-5</td>\n",
       "      <td>Cameron Joseph</td>\n",
       "      <td>culture, news, lgbtq, politics, journalism, vi...</td>\n",
       "      <td>Vice</td>\n",
       "      <td>2020</td>\n",
       "      <td>5</td>\n",
       "      <td>...</td>\n",
       "      <td>0.886</td>\n",
       "      <td>0.093</td>\n",
       "      <td>0.9900</td>\n",
       "      <td>0.153348</td>\n",
       "      <td>0.426474</td>\n",
       "      <td>kanye west has officially submitted signature...</td>\n",
       "      <td>[kanye, west, has, officially, submitted, sign...</td>\n",
       "      <td>[kanye, west, officially, submitted, signature...</td>\n",
       "      <td>[kany, west, offici, submit, signatur, appear,...</td>\n",
       "      <td>335</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32435</th>\n",
       "      <td>Kanye West Has a Senior GOP Strategist Helping...</td>\n",
       "      <td>“Would you help me get Kanye West on the ballo...</td>\n",
       "      <td>A top Colorado GOP strategist is helping Kanye...</td>\n",
       "      <td>https://www.vice.com/en_us/article/889kv5/kany...</td>\n",
       "      <td>2020-08-5</td>\n",
       "      <td>Cameron Joseph</td>\n",
       "      <td>culture, news, lgbtq, politics, journalism, vi...</td>\n",
       "      <td>Vice</td>\n",
       "      <td>2020</td>\n",
       "      <td>5</td>\n",
       "      <td>...</td>\n",
       "      <td>0.906</td>\n",
       "      <td>0.067</td>\n",
       "      <td>0.9526</td>\n",
       "      <td>0.096014</td>\n",
       "      <td>0.450302</td>\n",
       "      <td>a top colorado gop strategist is helping kanye...</td>\n",
       "      <td>[a, top, colorado, gop, strategist, is, helpin...</td>\n",
       "      <td>[top, colorado, gop, strategist, helping, kany...</td>\n",
       "      <td>[top, colorado, gop, strategist, help, kany, w...</td>\n",
       "      <td>322</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32436</th>\n",
       "      <td>A Malaysian Minister Was Caught Vaping in Parl...</td>\n",
       "      <td>Malaysia’s Foreign Affairs Minister tried to s...</td>\n",
       "      <td>A Malaysian politician has apologized for vapi...</td>\n",
       "      <td>https://www.vice.com/en_us/article/bv8pad/mala...</td>\n",
       "      <td>2020-08-7</td>\n",
       "      <td>Heather Chen</td>\n",
       "      <td>culture, news, lgbtq, politics, journalism, vi...</td>\n",
       "      <td>Vice</td>\n",
       "      <td>2020</td>\n",
       "      <td>7</td>\n",
       "      <td>...</td>\n",
       "      <td>0.920</td>\n",
       "      <td>0.034</td>\n",
       "      <td>-0.7635</td>\n",
       "      <td>0.041335</td>\n",
       "      <td>0.332055</td>\n",
       "      <td>a malaysian politician has apologized for vapi...</td>\n",
       "      <td>[a, malaysian, politician, has, apologized, fo...</td>\n",
       "      <td>[malaysian, politician, apologized, vaping, in...</td>\n",
       "      <td>[malaysian, politician, apolog, vape, insid, p...</td>\n",
       "      <td>146</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32437</th>\n",
       "      <td>Hackers Hijack Reddit Mod Accounts to Post Pro...</td>\n",
       "      <td>Hackers took control and defaced several large...</td>\n",
       "      <td>Hackers took control of more than a dozen subr...</td>\n",
       "      <td>https://www.vice.com/en_us/article/y3zx7g/hack...</td>\n",
       "      <td>2020-08-7</td>\n",
       "      <td>Lorenzo Franceschi-Bicchierai</td>\n",
       "      <td>culture, news, lgbtq, politics, journalism, vi...</td>\n",
       "      <td>Vice</td>\n",
       "      <td>2020</td>\n",
       "      <td>7</td>\n",
       "      <td>...</td>\n",
       "      <td>0.846</td>\n",
       "      <td>0.128</td>\n",
       "      <td>0.9782</td>\n",
       "      <td>0.356296</td>\n",
       "      <td>0.512593</td>\n",
       "      <td>hackers took control of more than a dozen subr...</td>\n",
       "      <td>[hackers, took, control, of, more, than, a, do...</td>\n",
       "      <td>[hackers, took, control, dozen, subreddits, in...</td>\n",
       "      <td>[hacker, took, control, dozen, subreddit, incl...</td>\n",
       "      <td>120</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32438</th>\n",
       "      <td>Charges Against 24 High-Profile Activists Part...</td>\n",
       "      <td>Human Rights Watch China director Sophie Richa...</td>\n",
       "      <td>Hong Kong handed down charges against two doze...</td>\n",
       "      <td>https://www.vice.com/en_us/article/7kpdaq/char...</td>\n",
       "      <td>2020-08-7</td>\n",
       "      <td>Sammy Westfall</td>\n",
       "      <td>culture, news, lgbtq, politics, journalism, vi...</td>\n",
       "      <td>Vice</td>\n",
       "      <td>2020</td>\n",
       "      <td>7</td>\n",
       "      <td>...</td>\n",
       "      <td>0.918</td>\n",
       "      <td>0.041</td>\n",
       "      <td>0.5994</td>\n",
       "      <td>0.082860</td>\n",
       "      <td>0.340667</td>\n",
       "      <td>hong kong handed down charges against two doze...</td>\n",
       "      <td>[hong, kong, handed, down, charges, against, t...</td>\n",
       "      <td>[hong, kong, handed, charges, two, dozen, pro-...</td>\n",
       "      <td>[hong, kong, hand, charg, two, dozen, pro-demo...</td>\n",
       "      <td>310</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>32439 rows × 25 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   Title  \\\n",
       "0                            What we're watching in 2020   \n",
       "1      Chief Justice Roberts says Americans may \"take...   \n",
       "2                               Trump's twin war threats   \n",
       "3      Pollak: Welcome to 2020, the Year of the Good ...   \n",
       "4      Warren Blames Unrest on Trump's 'Recklessness'...   \n",
       "...                                                  ...   \n",
       "32434  A Well-Connected GOP Strategist Is Helping Kan...   \n",
       "32435  Kanye West Has a Senior GOP Strategist Helping...   \n",
       "32436  A Malaysian Minister Was Caught Vaping in Parl...   \n",
       "32437  Hackers Hijack Reddit Mod Accounts to Post Pro...   \n",
       "32438  Charges Against 24 High-Profile Activists Part...   \n",
       "\n",
       "                                             Description  \\\n",
       "0      Trump's post-impeachment future, Big Tech regu...   \n",
       "1      It reads as a mission statement ahead of Presi...   \n",
       "2      They will truly test his diplomatic mix of blu...   \n",
       "3      The year 2020 begins with incredible potential...   \n",
       "4      Sen. Elizabeth Warren blamed President Trump f...   \n",
       "...                                                  ...   \n",
       "32434  Lane Ruhland filed West’s filing paperwork and...   \n",
       "32435  “Would you help me get Kanye West on the ballo...   \n",
       "32436  Malaysia’s Foreign Affairs Minister tried to s...   \n",
       "32437  Hackers took control and defaced several large...   \n",
       "32438  Human Rights Watch China director Sophie Richa...   \n",
       "\n",
       "                                                 Content  \\\n",
       "0      This is going to be a momentous presidential e...   \n",
       "1      Supreme Court Chief Justice John Roberts warne...   \n",
       "2      President Trump suddenly faces two global cris...   \n",
       "3      The year 2020 begins with incredible potential...   \n",
       "4      Sen. Elizabeth Warren (D-MA) on Tuesday blamed...   \n",
       "...                                                  ...   \n",
       "32434   Kanye West has officially submitted signature...   \n",
       "32435  A top Colorado GOP strategist is helping Kanye...   \n",
       "32436  A Malaysian politician has apologized for vapi...   \n",
       "32437  Hackers took control of more than a dozen subr...   \n",
       "32438  Hong Kong handed down charges against two doze...   \n",
       "\n",
       "                                                     URL        Date  \\\n",
       "0      https://www.axios.com/what-we-are-watching-202...  2020-01-01   \n",
       "1      https://www.axios.com/chief-justice-john-rober...  2020-01-01   \n",
       "2      https://www.axios.com/north-korea-iran-trump-w...  2020-01-01   \n",
       "3      https://www.breitbart.com/2020-election/2019/1...  2020-01-01   \n",
       "4      https://www.breitbart.com/middle-east/2019/12/...  2020-01-01   \n",
       "...                                                  ...         ...   \n",
       "32434  https://www.vice.com/en_us/article/akzy3b/a-we...   2020-08-5   \n",
       "32435  https://www.vice.com/en_us/article/889kv5/kany...   2020-08-5   \n",
       "32436  https://www.vice.com/en_us/article/bv8pad/mala...   2020-08-7   \n",
       "32437  https://www.vice.com/en_us/article/y3zx7g/hack...   2020-08-7   \n",
       "32438  https://www.vice.com/en_us/article/7kpdaq/char...   2020-08-7   \n",
       "\n",
       "                              Author  \\\n",
       "0            Jim VandeHei,Mike Allen   \n",
       "1                      Jacob Knutson   \n",
       "2            Mike Allen,Jim VandeHei   \n",
       "3                     Joel B. Pollak   \n",
       "4                       Hannah Bleau   \n",
       "...                              ...   \n",
       "32434                 Cameron Joseph   \n",
       "32435                 Cameron Joseph   \n",
       "32436                   Heather Chen   \n",
       "32437  Lorenzo Franceschi-Bicchierai   \n",
       "32438                 Sammy Westfall   \n",
       "\n",
       "                                                Category NewsProvider  Year  \\\n",
       "0                                      Politics & Policy        Axios  2020   \n",
       "1                                      Politics & Policy        Axios  2020   \n",
       "2                                      Politics & Policy        Axios  2020   \n",
       "3                           Politics,Media,2020 Election    Breitbart  2020   \n",
       "4        Politics,National Security,Israel / Middle East    Breitbart  2020   \n",
       "...                                                  ...          ...   ...   \n",
       "32434  culture, news, lgbtq, politics, journalism, vi...         Vice  2020   \n",
       "32435  culture, news, lgbtq, politics, journalism, vi...         Vice  2020   \n",
       "32436  culture, news, lgbtq, politics, journalism, vi...         Vice  2020   \n",
       "32437  culture, news, lgbtq, politics, journalism, vi...         Vice  2020   \n",
       "32438  culture, news, lgbtq, politics, journalism, vi...         Vice  2020   \n",
       "\n",
       "      Day  ... neuSentiment posSentiment compoundSentiment  polarity  \\\n",
       "0      01  ...        0.858        0.091            0.9834  0.168711   \n",
       "1      01  ...        0.796        0.149            0.9779  0.144646   \n",
       "2      01  ...        0.840        0.040           -0.9804  0.029773   \n",
       "3      01  ...        0.760        0.110           -0.9320  0.052025   \n",
       "4      01  ...        0.800        0.056           -0.9968 -0.020271   \n",
       "...    ..  ...          ...          ...               ...       ...   \n",
       "32434   5  ...        0.886        0.093            0.9900  0.153348   \n",
       "32435   5  ...        0.906        0.067            0.9526  0.096014   \n",
       "32436   7  ...        0.920        0.034           -0.7635  0.041335   \n",
       "32437   7  ...        0.846        0.128            0.9782  0.356296   \n",
       "32438   7  ...        0.918        0.041            0.5994  0.082860   \n",
       "\n",
       "       subjectivity                                      Content_clean  \\\n",
       "0          0.358021  this is going to be a momentous presidential e...   \n",
       "1          0.355859  supreme court chief justice john roberts warne...   \n",
       "2          0.298242  president trump suddenly faces two global cris...   \n",
       "3          0.445746  the year 2020 begins with incredible potential...   \n",
       "4          0.459366  sen. elizabeth warren (d-ma) on tuesday blamed...   \n",
       "...             ...                                                ...   \n",
       "32434      0.426474   kanye west has officially submitted signature...   \n",
       "32435      0.450302  a top colorado gop strategist is helping kanye...   \n",
       "32436      0.332055  a malaysian politician has apologized for vapi...   \n",
       "32437      0.512593  hackers took control of more than a dozen subr...   \n",
       "32438      0.340667  hong kong handed down charges against two doze...   \n",
       "\n",
       "                                           Content_words  \\\n",
       "0      [this, is, going, to, be, a, momentous, presid...   \n",
       "1      [supreme, court, chief, justice, john, roberts...   \n",
       "2      [president, trump, suddenly, faces, two, globa...   \n",
       "3      [the, year, 2020, begins, with, incredible, po...   \n",
       "4      [sen, elizabeth, warren, d-ma, on, tuesday, bl...   \n",
       "...                                                  ...   \n",
       "32434  [kanye, west, has, officially, submitted, sign...   \n",
       "32435  [a, top, colorado, gop, strategist, is, helpin...   \n",
       "32436  [a, malaysian, politician, has, apologized, fo...   \n",
       "32437  [hackers, took, control, of, more, than, a, do...   \n",
       "32438  [hong, kong, handed, down, charges, against, t...   \n",
       "\n",
       "                                   Content_filteredwords  \\\n",
       "0      [going, momentous, presidential, election, yea...   \n",
       "1      [supreme, court, chief, justice, john, roberts...   \n",
       "2      [president, trump, suddenly, faces, two, globa...   \n",
       "3      [year, 2020, begins, incredible, potential, su...   \n",
       "4      [sen, elizabeth, warren, d-ma, tuesday, blamed...   \n",
       "...                                                  ...   \n",
       "32434  [kanye, west, officially, submitted, signature...   \n",
       "32435  [top, colorado, gop, strategist, helping, kany...   \n",
       "32436  [malaysian, politician, apologized, vaping, in...   \n",
       "32437  [hackers, took, control, dozen, subreddits, in...   \n",
       "32438  [hong, kong, handed, charges, two, dozen, pro-...   \n",
       "\n",
       "                                    Content_stemmedwords  Totalwords  \n",
       "0      [go, moment, presidenti, elect, year, face, re...         393  \n",
       "1      [suprem, court, chief, justic, john, robert, w...         145  \n",
       "2      [presid, trump, suddenli, face, two, global, c...         169  \n",
       "3      [year, 2020, begin, incred, potenti, support, ...         240  \n",
       "4      [sen, elizabeth, warren, d-ma, tuesday, blame,...         390  \n",
       "...                                                  ...         ...  \n",
       "32434  [kany, west, offici, submit, signatur, appear,...         335  \n",
       "32435  [top, colorado, gop, strategist, help, kany, w...         322  \n",
       "32436  [malaysian, politician, apolog, vape, insid, p...         146  \n",
       "32437  [hacker, took, control, dozen, subreddit, incl...         120  \n",
       "32438  [hong, kong, hand, charg, two, dozen, pro-demo...         310  \n",
       "\n",
       "[32439 rows x 25 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Lowercase\n",
    "usnews[\"Content_clean\"] = usnews.apply(lambda x: x[\"Content\"].lower(), axis = \"columns\")\n",
    "\n",
    "# Delete whitespace\n",
    "whitespace_regex = r\"[\\t\\r\\n]\"\n",
    "usnews[\"Content_clean\"] = usnews.apply(lambda x: re.sub(whitespace_regex, \"\", x[\"Content_clean\"]), axis=\"columns\")\n",
    "\n",
    "# Tokenize words\n",
    "non_characters_regex = r\"[^a-zA-Z0-9 -]\"\n",
    "usnews[\"Content_words\"] = usnews.apply(lambda x: nltk.word_tokenize(re.sub(non_characters_regex, \"\", x[\"Content_clean\"])), axis = \"columns\")\n",
    "\n",
    "## Delete stopwords\n",
    "from nltk.corpus import stopwords\n",
    "stop_words = stopwords.words('english')\n",
    "# stop_words.append()\n",
    "# print(stop_words)\n",
    "usnews[\"Content_filteredwords\"] = usnews.apply(lambda x: [w for w in x[\"Content_words\"] if not w in stop_words], axis = \"columns\")\n",
    "\n",
    "# Word stemming\n",
    "from nltk.stem import PorterStemmer\n",
    "from nltk.tokenize import sent_tokenize, word_tokenize\n",
    "ps = PorterStemmer()\n",
    "def stemWords(words):\n",
    "    stemmed_words = []\n",
    "    for word in words:\n",
    "        stemmed_words.append(ps.stem(word))\n",
    "    return stemmed_words\n",
    "usnews[\"Content_stemmedwords\"] = usnews.apply(lambda x: stemWords(x[\"Content_filteredwords\"]), axis = \"columns\")\n",
    "\n",
    "# Count words\n",
    "usnews['Totalwords'] = [len(x) for x in usnews['Content_stemmedwords']]\n",
    "usnews"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LDA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\envs\\Patreon\\lib\\site-packages\\ipykernel\\ipkernel.py:287: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n",
      "  and should_run_async(code)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[(6,\n",
       "  '0.042*\"israel\" + 0.024*\"bass\" + 0.023*\"west\" + 0.021*\"antifa\" + 0.016*\"isra\" + 0.014*\"palestinian\" + 0.011*\"said\" + 0.011*\"netanyahu\" + 0.011*\"peac\" + 0.009*\"jewish\"'),\n",
       " (2,\n",
       "  '0.036*\"hous\" + 0.019*\"senat\" + 0.015*\"committe\" + 0.015*\"democrat\" + 0.015*\"bill\" + 0.012*\"lawmak\" + 0.012*\"member\" + 0.010*\"republican\" + 0.010*\"would\" + 0.010*\"said\"'),\n",
       " (1,\n",
       "  '0.012*\"eu\" + 0.011*\"govern\" + 0.010*\"uk\" + 0.010*\"london\" + 0.010*\"said\" + 0.010*\"countri\" + 0.010*\"minist\" + 0.009*\"european\" + 0.007*\"british\" + 0.007*\"would\"'),\n",
       " (4,\n",
       "  '0.029*\"rep\" + 0.022*\"district\" + 0.018*\"republican\" + 0.016*\"trump\" + 0.015*\"pm\" + 0.014*\"congression\" + 0.013*\"hous\" + 0.010*\"seat\" + 0.010*\"democrat\" + 0.010*\"say\"'),\n",
       " (9,\n",
       "  '0.034*\"china\" + 0.024*\"us\" + 0.020*\"chines\" + 0.015*\"state\" + 0.012*\"said\" + 0.011*\"unit\" + 0.010*\"countri\" + 0.010*\"world\" + 0.009*\"pompeo\" + 0.008*\"communist\"'),\n",
       " (10,\n",
       "  '0.043*\"biden\" + 0.019*\"campaign\" + 0.018*\"democrat\" + 0.016*\"trump\" + 0.011*\"joe\" + 0.011*\"presid\" + 0.009*\"parti\" + 0.008*\"former\" + 0.008*\"convent\" + 0.008*\"voter\"'),\n",
       " (11,\n",
       "  '0.014*\"said\" + 0.007*\"trump\" + 0.007*\"economi\" + 0.007*\"econom\" + 0.006*\"would\" + 0.006*\"busi\" + 0.005*\"us\" + 0.005*\"vaccin\" + 0.005*\"could\" + 0.005*\"govern\"'),\n",
       " (16,\n",
       "  '0.066*\"trump\" + 0.048*\"presid\" + 0.021*\"said\" + 0.014*\"white\" + 0.011*\"hous\" + 0.011*\"donald\" + 0.006*\"administr\" + 0.006*\"say\" + 0.006*\"nation\" + 0.005*\"news\"'),\n",
       " (15,\n",
       "  '0.014*\"school\" + 0.011*\"said\" + 0.011*\"state\" + 0.009*\"health\" + 0.009*\"mask\" + 0.008*\"peopl\" + 0.008*\"new\" + 0.006*\"public\" + 0.005*\"citi\" + 0.005*\"care\"'),\n",
       " (5,\n",
       "  '0.009*\"peopl\" + 0.007*\"one\" + 0.006*\"live\" + 0.006*\"like\" + 0.005*\"say\" + 0.005*\"trump\" + 0.005*\"american\" + 0.005*\"america\" + 0.005*\"black\" + 0.004*\"said\"')]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Prepare input for LDA model\n",
    "import scipy\n",
    "from gensim.corpora import Dictionary\n",
    "texts = usnews[\"Content_stemmedwords\"].to_list()\n",
    "dictionary = Dictionary(texts)\n",
    "corpus = [dictionary.doc2bow(text) for text in texts]\n",
    "\n",
    "# Compute LDA model\n",
    "from gensim.models import LdaModel\n",
    "ldamodel = LdaModel(corpus=corpus, num_topics= 20, id2word=dictionary, alpha = 'auto')\n",
    "ldamodel.show_topics()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Coherence Score:  0.5015015778174381\n"
     ]
    }
   ],
   "source": [
    "# Compute Coherence Score\n",
    "from gensim.models import CoherenceModel\n",
    "coherence_model_lda = CoherenceModel(model=ldamodel, texts=texts, dictionary=dictionary, coherence='c_v')\n",
    "coherence_lda = coherence_model_lda.get_coherence()\n",
    "print('\\nCoherence Score: ', coherence_lda)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save lda model\n",
    "#ldamodel.save('ldamodel/lda.model')\n",
    "\n",
    "# Load lda model\n",
    "#from gensim.models import LdaModel\n",
    "#ldamodel =  LdaModel.load('lda.model')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from gensim.models import LdaModel\n",
    "from gensim.models import CoherenceModel\n",
    "\n",
    "# Find optimal numbers of topics (highest coherence scores)\n",
    "def compute_coherence_values(dictionary, corpus, texts, limit, start=2, step=3):\n",
    "    \"\"\"\n",
    "    Compute c_v coherence for various number of topics\n",
    "\n",
    "    Parameters:\n",
    "    ----------\n",
    "    dictionary : Gensim dictionary\n",
    "    corpus : Gensim corpus\n",
    "    texts : List of input texts\n",
    "    limit : Max num of topics\n",
    "\n",
    "    Returns:\n",
    "    -------\n",
    "    model_list : List of LDA topic models\n",
    "    coherence_values : Coherence values corresponding to the LDA model with respective number of topics\n",
    "    \"\"\"\n",
    "    coherence_values = []\n",
    "    model_list = []\n",
    "    for num_topics in range(start, limit, step):\n",
    "        model = LdaModel(corpus=corpus, num_topics=num_topics, id2word=dictionary)\n",
    "        model_list.append(model)\n",
    "        coherencemodel = CoherenceModel(model=model, texts=texts, dictionary=dictionary, coherence='c_v')\n",
    "        coherence_values.append(coherencemodel.get_coherence())\n",
    "\n",
    "    return model_list, coherence_values\n",
    "\n",
    "model_list, coherence_values = compute_coherence_values(dictionary=dictionary, corpus=corpus, texts=texts, start=2, limit=40, step=6)\n",
    "\n",
    "# Print the coherence scores\n",
    "for m, cv in zip(x, coherence_values):\n",
    "    print(\"Num Topics =\", m, \" has Coherence Value of\", round(cv, 4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Document_No</th>\n",
       "      <th>Dominant_Topic</th>\n",
       "      <th>Topic_Perc_Contrib</th>\n",
       "      <th>Keywords</th>\n",
       "      <th>Text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>0.1909</td>\n",
       "      <td>said, health, test, china, viru, us, mask, sta...</td>\n",
       "      <td>[go, moment, presidenti, elect, year, face, re...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>8.0</td>\n",
       "      <td>0.2915</td>\n",
       "      <td>court, attorney, justic, case, judg, feder, in...</td>\n",
       "      <td>[suprem, court, chief, justic, john, robert, w...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.3441</td>\n",
       "      <td>protest, us, peac, hong, iran, kong, china, at...</td>\n",
       "      <td>[presid, trump, suddenli, face, two, global, c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.3016</td>\n",
       "      <td>trump, presid, said, white, administr, hous, a...</td>\n",
       "      <td>[year, 2020, begin, incred, potenti, support, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.4748</td>\n",
       "      <td>protest, us, peac, hong, iran, kong, china, at...</td>\n",
       "      <td>[sen, elizabeth, warren, d-ma, tuesday, blame,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.5361</td>\n",
       "      <td>protest, us, peac, hong, iran, kong, china, at...</td>\n",
       "      <td>[depart, defens, secretari, mark, esper, annou...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>6</td>\n",
       "      <td>18.0</td>\n",
       "      <td>0.6971</td>\n",
       "      <td>polic, citi, offic, said, protest, enforc, new...</td>\n",
       "      <td>[polic, lancast, ohio, releas, video, moment, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>7</td>\n",
       "      <td>15.0</td>\n",
       "      <td>0.6289</td>\n",
       "      <td>senat, trump, said, committe, hous, presid, re...</td>\n",
       "      <td>[speak, report, tuesday, trump, attorney, rudi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>8</td>\n",
       "      <td>19.0</td>\n",
       "      <td>0.4590</td>\n",
       "      <td>said, offici, us, secur, nation, depart, state...</td>\n",
       "      <td>[tuesday, broadcast, fox, news, channel, speci...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>9</td>\n",
       "      <td>14.0</td>\n",
       "      <td>0.5691</td>\n",
       "      <td>live, black, peopl, right, american, one, said...</td>\n",
       "      <td>[greyhound, partnership, nation, runaway, safe...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Document_No  Dominant_Topic  Topic_Perc_Contrib  \\\n",
       "0            0            10.0              0.1909   \n",
       "1            1             8.0              0.2915   \n",
       "2            2             5.0              0.3441   \n",
       "3            3             3.0              0.3016   \n",
       "4            4             5.0              0.4748   \n",
       "5            5             5.0              0.5361   \n",
       "6            6            18.0              0.6971   \n",
       "7            7            15.0              0.6289   \n",
       "8            8            19.0              0.4590   \n",
       "9            9            14.0              0.5691   \n",
       "\n",
       "                                            Keywords  \\\n",
       "0  said, health, test, china, viru, us, mask, sta...   \n",
       "1  court, attorney, justic, case, judg, feder, in...   \n",
       "2  protest, us, peac, hong, iran, kong, china, at...   \n",
       "3  trump, presid, said, white, administr, hous, a...   \n",
       "4  protest, us, peac, hong, iran, kong, china, at...   \n",
       "5  protest, us, peac, hong, iran, kong, china, at...   \n",
       "6  polic, citi, offic, said, protest, enforc, new...   \n",
       "7  senat, trump, said, committe, hous, presid, re...   \n",
       "8  said, offici, us, secur, nation, depart, state...   \n",
       "9  live, black, peopl, right, american, one, said...   \n",
       "\n",
       "                                                Text  \n",
       "0  [go, moment, presidenti, elect, year, face, re...  \n",
       "1  [suprem, court, chief, justic, john, robert, w...  \n",
       "2  [presid, trump, suddenli, face, two, global, c...  \n",
       "3  [year, 2020, begin, incred, potenti, support, ...  \n",
       "4  [sen, elizabeth, warren, d-ma, tuesday, blame,...  \n",
       "5  [depart, defens, secretari, mark, esper, annou...  \n",
       "6  [polic, lancast, ohio, releas, video, moment, ...  \n",
       "7  [speak, report, tuesday, trump, attorney, rudi...  \n",
       "8  [tuesday, broadcast, fox, news, channel, speci...  \n",
       "9  [greyhound, partnership, nation, runaway, safe...  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Assign main topic to each document\n",
    "def format_topics_sentences(ldamodel, corpus, texts):\n",
    "    # Init output\n",
    "    sent_topics_df = pd.DataFrame()\n",
    "\n",
    "    # Get main topic in each document\n",
    "    for i, row in enumerate(ldamodel[corpus]):\n",
    "        row = sorted(row, key=lambda x: (x[1]), reverse=True)\n",
    "        # Get the Dominant topic, Perc Contribution and Keywords for each document\n",
    "        for j, (topic_num, prop_topic) in enumerate(row):\n",
    "            if j == 0:  # => dominant topic\n",
    "                wp = ldamodel.show_topic(topic_num)\n",
    "                topic_keywords = \", \".join([word for word, prop in wp])\n",
    "                sent_topics_df = sent_topics_df.append(pd.Series([int(topic_num), round(prop_topic,4), topic_keywords]), ignore_index=True)\n",
    "            else:\n",
    "                break\n",
    "    sent_topics_df.columns = ['Dominant_Topic', 'Perc_Contribution', 'Topic_Keywords']\n",
    "\n",
    "    # Add original text to the end of the output\n",
    "    contents = pd.Series(texts)\n",
    "    sent_topics_df = pd.concat([sent_topics_df, contents], axis=1)\n",
    "    return(sent_topics_df)\n",
    "\n",
    "df_topic_sents_keywords = format_topics_sentences(ldamodel=ldamodel, corpus=corpus, texts=texts)\n",
    "\n",
    "# Format\n",
    "df_dominant_topic = df_topic_sents_keywords.reset_index()\n",
    "df_dominant_topic.columns = ['Document_No', 'Dominant_Topic', 'Topic_Perc_Contrib', 'Keywords', 'Text']\n",
    "df_dominant_topic.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Title</th>\n",
       "      <th>Description</th>\n",
       "      <th>Content</th>\n",
       "      <th>URL</th>\n",
       "      <th>Date</th>\n",
       "      <th>Author</th>\n",
       "      <th>Category</th>\n",
       "      <th>NewsProvider</th>\n",
       "      <th>Year</th>\n",
       "      <th>Day</th>\n",
       "      <th>...</th>\n",
       "      <th>Text</th>\n",
       "      <th>Dominant_Topic</th>\n",
       "      <th>Topic_Perc_Contrib</th>\n",
       "      <th>Keywords</th>\n",
       "      <th>Text</th>\n",
       "      <th>Document_No</th>\n",
       "      <th>Dominant_Topic</th>\n",
       "      <th>Topic_Perc_Contrib</th>\n",
       "      <th>Keywords</th>\n",
       "      <th>Text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>What we're watching in 2020</td>\n",
       "      <td>Trump's post-impeachment future, Big Tech regu...</td>\n",
       "      <td>This is going to be a momentous presidential e...</td>\n",
       "      <td>https://www.axios.com/what-we-are-watching-202...</td>\n",
       "      <td>2020-01-01</td>\n",
       "      <td>Jim VandeHei,Mike Allen</td>\n",
       "      <td>Politics &amp; Policy</td>\n",
       "      <td>Axios</td>\n",
       "      <td>2020</td>\n",
       "      <td>01</td>\n",
       "      <td>...</td>\n",
       "      <td>[go, moment, presidenti, elect, year, face, re...</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.2675</td>\n",
       "      <td>said, health, vaccin, like, use, work, would, ...</td>\n",
       "      <td>[go, moment, presidenti, elect, year, face, re...</td>\n",
       "      <td>0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>0.1909</td>\n",
       "      <td>said, health, test, china, viru, us, mask, sta...</td>\n",
       "      <td>[go, moment, presidenti, elect, year, face, re...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Chief Justice Roberts says Americans may \"take...</td>\n",
       "      <td>It reads as a mission statement ahead of Presi...</td>\n",
       "      <td>Supreme Court Chief Justice John Roberts warne...</td>\n",
       "      <td>https://www.axios.com/chief-justice-john-rober...</td>\n",
       "      <td>2020-01-01</td>\n",
       "      <td>Jacob Knutson</td>\n",
       "      <td>Politics &amp; Policy</td>\n",
       "      <td>Axios</td>\n",
       "      <td>2020</td>\n",
       "      <td>01</td>\n",
       "      <td>...</td>\n",
       "      <td>[suprem, court, chief, justic, john, robert, w...</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.5649</td>\n",
       "      <td>court, justic, feder, attorney, law, case, sai...</td>\n",
       "      <td>[suprem, court, chief, justic, john, robert, w...</td>\n",
       "      <td>1</td>\n",
       "      <td>8.0</td>\n",
       "      <td>0.2915</td>\n",
       "      <td>court, attorney, justic, case, judg, feder, in...</td>\n",
       "      <td>[suprem, court, chief, justic, john, robert, w...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Trump's twin war threats</td>\n",
       "      <td>They will truly test his diplomatic mix of blu...</td>\n",
       "      <td>President Trump suddenly faces two global cris...</td>\n",
       "      <td>https://www.axios.com/north-korea-iran-trump-w...</td>\n",
       "      <td>2020-01-01</td>\n",
       "      <td>Mike Allen,Jim VandeHei</td>\n",
       "      <td>Politics &amp; Policy</td>\n",
       "      <td>Axios</td>\n",
       "      <td>2020</td>\n",
       "      <td>01</td>\n",
       "      <td>...</td>\n",
       "      <td>[presid, trump, suddenli, face, two, global, c...</td>\n",
       "      <td>18.0</td>\n",
       "      <td>0.4016</td>\n",
       "      <td>polic, protest, offic, said, citi, peopl, fire...</td>\n",
       "      <td>[presid, trump, suddenli, face, two, global, c...</td>\n",
       "      <td>2</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.3441</td>\n",
       "      <td>protest, us, peac, hong, iran, kong, china, at...</td>\n",
       "      <td>[presid, trump, suddenli, face, two, global, c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Pollak: Welcome to 2020, the Year of the Good ...</td>\n",
       "      <td>The year 2020 begins with incredible potential...</td>\n",
       "      <td>The year 2020 begins with incredible potential...</td>\n",
       "      <td>https://www.breitbart.com/2020-election/2019/1...</td>\n",
       "      <td>2020-01-01</td>\n",
       "      <td>Joel B. Pollak</td>\n",
       "      <td>Politics,Media,2020 Election</td>\n",
       "      <td>Breitbart</td>\n",
       "      <td>2020</td>\n",
       "      <td>01</td>\n",
       "      <td>...</td>\n",
       "      <td>[year, 2020, begin, incred, potenti, support, ...</td>\n",
       "      <td>7.0</td>\n",
       "      <td>0.2426</td>\n",
       "      <td>peopl, polit, countri, american, us, one, part...</td>\n",
       "      <td>[year, 2020, begin, incred, potenti, support, ...</td>\n",
       "      <td>3</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.3016</td>\n",
       "      <td>trump, presid, said, white, administr, hous, a...</td>\n",
       "      <td>[year, 2020, begin, incred, potenti, support, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Warren Blames Unrest on Trump's 'Recklessness'...</td>\n",
       "      <td>Sen. Elizabeth Warren blamed President Trump f...</td>\n",
       "      <td>Sen. Elizabeth Warren (D-MA) on Tuesday blamed...</td>\n",
       "      <td>https://www.breitbart.com/middle-east/2019/12/...</td>\n",
       "      <td>2020-01-01</td>\n",
       "      <td>Hannah Bleau</td>\n",
       "      <td>Politics,National Security,Israel / Middle East</td>\n",
       "      <td>Breitbart</td>\n",
       "      <td>2020</td>\n",
       "      <td>01</td>\n",
       "      <td>...</td>\n",
       "      <td>[sen, elizabeth, warren, d-ma, tuesday, blame,...</td>\n",
       "      <td>18.0</td>\n",
       "      <td>0.4976</td>\n",
       "      <td>polic, protest, offic, said, citi, peopl, fire...</td>\n",
       "      <td>[sen, elizabeth, warren, d-ma, tuesday, blame,...</td>\n",
       "      <td>4</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.4748</td>\n",
       "      <td>protest, us, peac, hong, iran, kong, china, at...</td>\n",
       "      <td>[sen, elizabeth, warren, d-ma, tuesday, blame,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31985</th>\n",
       "      <td>The Best Way to Help Schools Reopen</td>\n",
       "      <td>Educators aren’t public health experts. If we ...</td>\n",
       "      <td>The unfolding complexity of school reopening r...</td>\n",
       "      <td>https://www.politico.com/news/agenda/2020/08/2...</td>\n",
       "      <td>2020-08-27</td>\n",
       "      <td>MARIO RAMIREZ</td>\n",
       "      <td>OPINION</td>\n",
       "      <td>Politico</td>\n",
       "      <td>2020</td>\n",
       "      <td>27</td>\n",
       "      <td>...</td>\n",
       "      <td>[unfold, complex, school, reopen, reflect, sim...</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.3998</td>\n",
       "      <td>said, health, vaccin, like, use, work, would, ...</td>\n",
       "      <td>[unfold, complex, school, reopen, reflect, sim...</td>\n",
       "      <td>31985</td>\n",
       "      <td>17.0</td>\n",
       "      <td>0.4629</td>\n",
       "      <td>state, school, vote, elect, ballot, said, stud...</td>\n",
       "      <td>[unfold, complex, school, reopen, reflect, sim...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31986</th>\n",
       "      <td>How Trump Mastered the Art of Telling History ...</td>\n",
       "      <td>The Republican convention has glossed over the...</td>\n",
       "      <td>Donald Trump, according to the first three nig...</td>\n",
       "      <td>https://www.politico.com/news/magazine/2020/08...</td>\n",
       "      <td>2020-08-27</td>\n",
       "      <td>MICHAEL KRUSE</td>\n",
       "      <td>OPINION</td>\n",
       "      <td>Politico</td>\n",
       "      <td>2020</td>\n",
       "      <td>27</td>\n",
       "      <td>...</td>\n",
       "      <td>[donald, trump, accord, first, three, night, r...</td>\n",
       "      <td>10.0</td>\n",
       "      <td>0.2054</td>\n",
       "      <td>presid, trump, said, go, think, say, peopl, kn...</td>\n",
       "      <td>[donald, trump, accord, first, three, night, r...</td>\n",
       "      <td>31986</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.3932</td>\n",
       "      <td>trump, presid, said, white, administr, hous, a...</td>\n",
       "      <td>[donald, trump, accord, first, three, night, r...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31987</th>\n",
       "      <td>Bush, McCain and Romney presidential staffers ...</td>\n",
       "      <td>“We know Mitt and Joe share those same essenti...</td>\n",
       "      <td>Support for Joe Biden's White House bid is gro...</td>\n",
       "      <td>https://www.politico.com/news/2020/08/27/mitt-...</td>\n",
       "      <td>2020-08-27</td>\n",
       "      <td>MAX COHEN</td>\n",
       "      <td>2020 Elections</td>\n",
       "      <td>Politico</td>\n",
       "      <td>2020</td>\n",
       "      <td>27</td>\n",
       "      <td>...</td>\n",
       "      <td>[support, joe, biden, white, hous, bid, grow, ...</td>\n",
       "      <td>11.0</td>\n",
       "      <td>0.4915</td>\n",
       "      <td>biden, campaign, democrat, said, joe, former, ...</td>\n",
       "      <td>[support, joe, biden, white, hous, bid, grow, ...</td>\n",
       "      <td>31987</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.5023</td>\n",
       "      <td>biden, democrat, campaign, elect, joe, parti, ...</td>\n",
       "      <td>[support, joe, biden, white, hous, bid, grow, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31988</th>\n",
       "      <td>GOP takes a third run at defining Republicans ...</td>\n",
       "      <td>After two days of dueling and at-time contradi...</td>\n",
       "      <td>On Monday, the only Black GOP senator sought t...</td>\n",
       "      <td>https://www.politico.com/news/2020/08/27/gop-h...</td>\n",
       "      <td>2020-08-27</td>\n",
       "      <td>GABBY ORR</td>\n",
       "      <td>2020 Conventions</td>\n",
       "      <td>Politico</td>\n",
       "      <td>2020</td>\n",
       "      <td>27</td>\n",
       "      <td>...</td>\n",
       "      <td>[monday, black, gop, senat, sought, convinc, a...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.2615</td>\n",
       "      <td>trump, presid, white, hous, said, donald, amer...</td>\n",
       "      <td>[monday, black, gop, senat, sought, convinc, a...</td>\n",
       "      <td>31988</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.3846</td>\n",
       "      <td>trump, presid, said, white, administr, hous, a...</td>\n",
       "      <td>[monday, black, gop, senat, sought, convinc, a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31989</th>\n",
       "      <td>‘It’s playing into Trump’s hands’: Dems fear s...</td>\n",
       "      <td>Some Wisconsin Democrats worry that the images...</td>\n",
       "      <td>KENOSHA, Wis. — Downtown buildings set ablaze ...</td>\n",
       "      <td>https://www.politico.com/news/2020/08/27/its-p...</td>\n",
       "      <td>2020-08-27</td>\n",
       "      <td>NATASHA KORECKI</td>\n",
       "      <td>2020 Elections</td>\n",
       "      <td>Politico</td>\n",
       "      <td>2020</td>\n",
       "      <td>27</td>\n",
       "      <td>...</td>\n",
       "      <td>[kenosha, wi, downtown, build, set, ablaz, ars...</td>\n",
       "      <td>18.0</td>\n",
       "      <td>0.2672</td>\n",
       "      <td>polic, protest, offic, said, citi, peopl, fire...</td>\n",
       "      <td>[kenosha, wi, downtown, build, set, ablaz, ars...</td>\n",
       "      <td>31989</td>\n",
       "      <td>18.0</td>\n",
       "      <td>0.3064</td>\n",
       "      <td>polic, citi, offic, said, protest, enforc, new...</td>\n",
       "      <td>[kenosha, wi, downtown, build, set, ablaz, ars...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>31990 rows × 42 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   Title  \\\n",
       "0                            What we're watching in 2020   \n",
       "1      Chief Justice Roberts says Americans may \"take...   \n",
       "2                               Trump's twin war threats   \n",
       "3      Pollak: Welcome to 2020, the Year of the Good ...   \n",
       "4      Warren Blames Unrest on Trump's 'Recklessness'...   \n",
       "...                                                  ...   \n",
       "31985               The Best Way to Help Schools Reopen    \n",
       "31986  How Trump Mastered the Art of Telling History ...   \n",
       "31987  Bush, McCain and Romney presidential staffers ...   \n",
       "31988  GOP takes a third run at defining Republicans ...   \n",
       "31989  ‘It’s playing into Trump’s hands’: Dems fear s...   \n",
       "\n",
       "                                             Description  \\\n",
       "0      Trump's post-impeachment future, Big Tech regu...   \n",
       "1      It reads as a mission statement ahead of Presi...   \n",
       "2      They will truly test his diplomatic mix of blu...   \n",
       "3      The year 2020 begins with incredible potential...   \n",
       "4      Sen. Elizabeth Warren blamed President Trump f...   \n",
       "...                                                  ...   \n",
       "31985  Educators aren’t public health experts. If we ...   \n",
       "31986  The Republican convention has glossed over the...   \n",
       "31987  “We know Mitt and Joe share those same essenti...   \n",
       "31988  After two days of dueling and at-time contradi...   \n",
       "31989  Some Wisconsin Democrats worry that the images...   \n",
       "\n",
       "                                                 Content  \\\n",
       "0      This is going to be a momentous presidential e...   \n",
       "1      Supreme Court Chief Justice John Roberts warne...   \n",
       "2      President Trump suddenly faces two global cris...   \n",
       "3      The year 2020 begins with incredible potential...   \n",
       "4      Sen. Elizabeth Warren (D-MA) on Tuesday blamed...   \n",
       "...                                                  ...   \n",
       "31985  The unfolding complexity of school reopening r...   \n",
       "31986  Donald Trump, according to the first three nig...   \n",
       "31987  Support for Joe Biden's White House bid is gro...   \n",
       "31988  On Monday, the only Black GOP senator sought t...   \n",
       "31989  KENOSHA, Wis. — Downtown buildings set ablaze ...   \n",
       "\n",
       "                                                     URL        Date  \\\n",
       "0      https://www.axios.com/what-we-are-watching-202...  2020-01-01   \n",
       "1      https://www.axios.com/chief-justice-john-rober...  2020-01-01   \n",
       "2      https://www.axios.com/north-korea-iran-trump-w...  2020-01-01   \n",
       "3      https://www.breitbart.com/2020-election/2019/1...  2020-01-01   \n",
       "4      https://www.breitbart.com/middle-east/2019/12/...  2020-01-01   \n",
       "...                                                  ...         ...   \n",
       "31985  https://www.politico.com/news/agenda/2020/08/2...  2020-08-27   \n",
       "31986  https://www.politico.com/news/magazine/2020/08...  2020-08-27   \n",
       "31987  https://www.politico.com/news/2020/08/27/mitt-...  2020-08-27   \n",
       "31988  https://www.politico.com/news/2020/08/27/gop-h...  2020-08-27   \n",
       "31989  https://www.politico.com/news/2020/08/27/its-p...  2020-08-27   \n",
       "\n",
       "                        Author  \\\n",
       "0      Jim VandeHei,Mike Allen   \n",
       "1                Jacob Knutson   \n",
       "2      Mike Allen,Jim VandeHei   \n",
       "3               Joel B. Pollak   \n",
       "4                 Hannah Bleau   \n",
       "...                        ...   \n",
       "31985            MARIO RAMIREZ   \n",
       "31986            MICHAEL KRUSE   \n",
       "31987                MAX COHEN   \n",
       "31988                GABBY ORR   \n",
       "31989          NATASHA KORECKI   \n",
       "\n",
       "                                              Category NewsProvider  Year Day  \\\n",
       "0                                    Politics & Policy        Axios  2020  01   \n",
       "1                                    Politics & Policy        Axios  2020  01   \n",
       "2                                    Politics & Policy        Axios  2020  01   \n",
       "3                         Politics,Media,2020 Election    Breitbart  2020  01   \n",
       "4      Politics,National Security,Israel / Middle East    Breitbart  2020  01   \n",
       "...                                                ...          ...   ...  ..   \n",
       "31985                                          OPINION     Politico  2020  27   \n",
       "31986                                          OPINION     Politico  2020  27   \n",
       "31987                                   2020 Elections     Politico  2020  27   \n",
       "31988                                 2020 Conventions     Politico  2020  27   \n",
       "31989                                  2020 Elections      Politico  2020  27   \n",
       "\n",
       "       ...                                               Text  Dominant_Topic  \\\n",
       "0      ...  [go, moment, presidenti, elect, year, face, re...             3.0   \n",
       "1      ...  [suprem, court, chief, justic, john, robert, w...             6.0   \n",
       "2      ...  [presid, trump, suddenli, face, two, global, c...            18.0   \n",
       "3      ...  [year, 2020, begin, incred, potenti, support, ...             7.0   \n",
       "4      ...  [sen, elizabeth, warren, d-ma, tuesday, blame,...            18.0   \n",
       "...    ...                                                ...             ...   \n",
       "31985  ...  [unfold, complex, school, reopen, reflect, sim...             3.0   \n",
       "31986  ...  [donald, trump, accord, first, three, night, r...            10.0   \n",
       "31987  ...  [support, joe, biden, white, hous, bid, grow, ...            11.0   \n",
       "31988  ...  [monday, black, gop, senat, sought, convinc, a...             0.0   \n",
       "31989  ...  [kenosha, wi, downtown, build, set, ablaz, ars...            18.0   \n",
       "\n",
       "       Topic_Perc_Contrib                                           Keywords  \\\n",
       "0                  0.2675  said, health, vaccin, like, use, work, would, ...   \n",
       "1                  0.5649  court, justic, feder, attorney, law, case, sai...   \n",
       "2                  0.4016  polic, protest, offic, said, citi, peopl, fire...   \n",
       "3                  0.2426  peopl, polit, countri, american, us, one, part...   \n",
       "4                  0.4976  polic, protest, offic, said, citi, peopl, fire...   \n",
       "...                   ...                                                ...   \n",
       "31985              0.3998  said, health, vaccin, like, use, work, would, ...   \n",
       "31986              0.2054  presid, trump, said, go, think, say, peopl, kn...   \n",
       "31987              0.4915  biden, campaign, democrat, said, joe, former, ...   \n",
       "31988              0.2615  trump, presid, white, hous, said, donald, amer...   \n",
       "31989              0.2672  polic, protest, offic, said, citi, peopl, fire...   \n",
       "\n",
       "                                                    Text  Document_No  \\\n",
       "0      [go, moment, presidenti, elect, year, face, re...            0   \n",
       "1      [suprem, court, chief, justic, john, robert, w...            1   \n",
       "2      [presid, trump, suddenli, face, two, global, c...            2   \n",
       "3      [year, 2020, begin, incred, potenti, support, ...            3   \n",
       "4      [sen, elizabeth, warren, d-ma, tuesday, blame,...            4   \n",
       "...                                                  ...          ...   \n",
       "31985  [unfold, complex, school, reopen, reflect, sim...        31985   \n",
       "31986  [donald, trump, accord, first, three, night, r...        31986   \n",
       "31987  [support, joe, biden, white, hous, bid, grow, ...        31987   \n",
       "31988  [monday, black, gop, senat, sought, convinc, a...        31988   \n",
       "31989  [kenosha, wi, downtown, build, set, ablaz, ars...        31989   \n",
       "\n",
       "       Dominant_Topic  Topic_Perc_Contrib  \\\n",
       "0                10.0              0.1909   \n",
       "1                 8.0              0.2915   \n",
       "2                 5.0              0.3441   \n",
       "3                 3.0              0.3016   \n",
       "4                 5.0              0.4748   \n",
       "...               ...                 ...   \n",
       "31985            17.0              0.4629   \n",
       "31986             3.0              0.3932   \n",
       "31987             6.0              0.5023   \n",
       "31988             3.0              0.3846   \n",
       "31989            18.0              0.3064   \n",
       "\n",
       "                                                Keywords  \\\n",
       "0      said, health, test, china, viru, us, mask, sta...   \n",
       "1      court, attorney, justic, case, judg, feder, in...   \n",
       "2      protest, us, peac, hong, iran, kong, china, at...   \n",
       "3      trump, presid, said, white, administr, hous, a...   \n",
       "4      protest, us, peac, hong, iran, kong, china, at...   \n",
       "...                                                  ...   \n",
       "31985  state, school, vote, elect, ballot, said, stud...   \n",
       "31986  trump, presid, said, white, administr, hous, a...   \n",
       "31987  biden, democrat, campaign, elect, joe, parti, ...   \n",
       "31988  trump, presid, said, white, administr, hous, a...   \n",
       "31989  polic, citi, offic, said, protest, enforc, new...   \n",
       "\n",
       "                                                    Text  \n",
       "0      [go, moment, presidenti, elect, year, face, re...  \n",
       "1      [suprem, court, chief, justic, john, robert, w...  \n",
       "2      [presid, trump, suddenli, face, two, global, c...  \n",
       "3      [year, 2020, begin, incred, potenti, support, ...  \n",
       "4      [sen, elizabeth, warren, d-ma, tuesday, blame,...  \n",
       "...                                                  ...  \n",
       "31985  [unfold, complex, school, reopen, reflect, sim...  \n",
       "31986  [donald, trump, accord, first, three, night, r...  \n",
       "31987  [support, joe, biden, white, hous, bid, grow, ...  \n",
       "31988  [monday, black, gop, senat, sought, convinc, a...  \n",
       "31989  [kenosha, wi, downtown, build, set, ablaz, ars...  \n",
       "\n",
       "[31990 rows x 42 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "usnews_final = pd.concat([usnews.reset_index(drop=True), df_dominant_topic],  axis = 1, sort = False)\n",
    "usnews_final"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "del usnews_final[\"Document_No\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "usnews_final.to_pickle(\"data/usnews.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\envs\\Patreon\\lib\\site-packages\\ipykernel\\ipkernel.py:287: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n",
      "  and should_run_async(code)\n"
     ]
    }
   ],
   "source": [
    "#import pyLDAvis.gensim\n",
    "#pyLDAvis.enable_notebook()\n",
    "#p = pyLDAvis.gensim.prepare(ldamodel, corpus, dictionary)\n",
    "#pyLDAvis.save_html(p, r'C:\\Users\\Jonathan Ratschat\\Google Drive\\Colab Notebooks\\Scraping\\usnews\\pyLDAvis.html')\n",
    "#pyLDAvis.display(p)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Not used in final analysis"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "# Load vice data sets\n",
    "vice_election = pd.read_csv(\"data/vice2020election.csv\")\n",
    "vice_politics = pd.read_csv(\"data/vicepolitics.csv\")\n",
    "\n",
    "# Merge date sets\n",
    "vice = vice_election.append(vice_politics)\n",
    "\n",
    "# Add column that indicates news provider\n",
    "vice[\"NewsProvider\"] = \"Vice\" \n",
    "\n",
    "# Drop articles with the same header\n",
    "vice = vice.drop_duplicates(subset='Title', keep=\"first\")\n",
    "\n",
    "# Drop articles that have no content\n",
    "vice = vice[~vice[\"Content\"].isnull()]\n",
    "\n",
    "# Exchange month names with numbers\n",
    "months = [\"January\", \"February\", \"March\", \"April\", \"May\", \"June\", \"July\", \"August\", \"December\"]\n",
    "replaced_months = [\"01,\", \"02,\", \"03,\", \"04,\", \"05,\", \"06,\", \"07,\", \"08,\", \"09,\"]\n",
    "for month, replaced_month in zip(months, replaced_months):\n",
    "    vice[\"Date\"] = vice.Date.str.replace(month, replaced_month)\n",
    "\n",
    "# Create new year, day, and month columns\n",
    "vice['Date'] = vice.apply(lambda x: x[\"Date\"][:12], axis = \"columns\")\n",
    "vice['Year'] = vice.apply(lambda x: x[\"Date\"][6:], axis = \"columns\")\n",
    "vice['Day'] = vice.apply(lambda x: x[\"Date\"][4:6], axis = \"columns\")\n",
    "vice['Month'] = vice.apply(lambda x: x[\"Date\"][:3], axis = \"columns\")\n",
    "\n",
    "# Clean , and whitespace\n",
    "vice[\"Year\"] = vice.apply(lambda x: x[\"Year\"].replace(\",\", \"\"), axis = 1)\n",
    "vice[\"Day\"] = vice.apply(lambda x: x[\"Day\"].replace(\",\", \"\"), axis = 1)\n",
    "vice[\"Month\"] = vice.apply(lambda x: x[\"Month\"].replace(\",\", \"\"), axis = 1)\n",
    "vice[\"Year\"] = vice.apply(lambda x: x[\"Year\"].replace(\" \", \"\"), axis = 1)\n",
    "vice[\"Day\"] = vice.apply(lambda x: x[\"Day\"].replace(\" \", \"\"), axis = 1)\n",
    "vice[\"Month\"] = vice.apply(lambda x: x[\"Month\"].replace(\" \", \"\"), axis = 1)\n",
    "\n",
    "# Create clean date column\n",
    "vice['Date'] = vice.apply(lambda x: x[\"Year\"] + \"-\" + x[\"Month\"] + \"-\" + x[\"Day\"], axis = \"columns\")\n",
    "\n",
    "# Exlude December (quick and dirty)\n",
    "vice = vice[vice[\"Year\"] != \"2019\"]\n",
    "\n",
    "del vice[\"Year\"]\n",
    "del vice[\"Day\"]\n",
    "del vice[\"Month\"]\n",
    "\n",
    "display(vice.tail(1))\n",
    "display(vice.count())"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}