02_download_transcripts.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "cfc4163e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import requests\n",
    "import time\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "3db4e8c9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>imageUrl</th>\n",
       "      <th>title</th>\n",
       "      <th>description</th>\n",
       "      <th>url</th>\n",
       "      <th>publicationDate</th>\n",
       "      <th>lastPublishedDate</th>\n",
       "      <th>category</th>\n",
       "      <th>isBreaking</th>\n",
       "      <th>isLive</th>\n",
       "      <th>duration</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>https://a57.foxnews.com/cf-images.us-east-1.pr...</td>\n",
       "      <td>'Fox News Sunday' on May 29, 2022</td>\n",
       "      <td>\"Fox News Sunday\" welcomed guests Mo Brooks, O...</td>\n",
       "      <td>/transcript/fox-news-sunday-on-may-29-2022</td>\n",
       "      <td>2022-05-29T14:08:01-04:00</td>\n",
       "      <td>2022-05-29T14:08:01-04:00</td>\n",
       "      <td>{'name': 'TRANSCRIPT', 'url': '/category/shows...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>https://a57.foxnews.com/cf-images.us-east-1.pr...</td>\n",
       "      <td>'Fox News Sunday' on May 22, 2022</td>\n",
       "      <td>\"Fox News Sunday\" welcomed guests Brian Deese,...</td>\n",
       "      <td>/transcript/fox-news-sunday-on-may-22-2022</td>\n",
       "      <td>2022-05-22T13:59:09-04:00</td>\n",
       "      <td>2022-05-22T13:59:09-04:00</td>\n",
       "      <td>{'name': 'TRANSCRIPT', 'url': '/category/shows...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>https://a57.foxnews.com/cf-images.us-east-1.pr...</td>\n",
       "      <td>'Fox News Sunday' on May 15, 2022</td>\n",
       "      <td>\"Fox News Sunday\" welcomed guests Kevin Stitt,...</td>\n",
       "      <td>/transcript/fox-news-sunday-on-may-15-2022</td>\n",
       "      <td>2022-05-15T13:59:21-04:00</td>\n",
       "      <td>2022-05-15T13:59:21-04:00</td>\n",
       "      <td>{'name': 'TRANSCRIPT', 'url': '/category/shows...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>https://a57.foxnews.com/cf-images.us-east-1.pr...</td>\n",
       "      <td>‘Fox News Sunday’ on April 17, 2022</td>\n",
       "      <td>\"Fox News Sunday\" welcomed guests Josh Krausha...</td>\n",
       "      <td>/transcript/fox-news-sunday-on-april-17-2022</td>\n",
       "      <td>2022-04-17T14:08:36-04:00</td>\n",
       "      <td>2022-04-17T14:08:36-04:00</td>\n",
       "      <td>{'name': 'TRANSCRIPT', 'url': '/category/shows...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>https://a57.foxnews.com/cf-images.us-east-1.pr...</td>\n",
       "      <td>'Fox News Sunday' on April 10, 2022</td>\n",
       "      <td>\"Fox News Sunday\" welcomed guests Mitch McConn...</td>\n",
       "      <td>/shows/fox-news-sunday/fox-news-sunday-on-apri...</td>\n",
       "      <td>2022-04-10T13:59:48-04:00</td>\n",
       "      <td>2022-04-10T13:59:48-04:00</td>\n",
       "      <td>{'name': 'TRANSCRIPT', 'url': '/category/shows...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32258</th>\n",
       "      <td>https://a57.foxnews.com///static.foxnews.com/s...</td>\n",
       "      <td>Reince Priebus on Trump's stance on Cuba, Cabi...</td>\n",
       "      <td>Incoming White House chief of staff weighs in ...</td>\n",
       "      <td>/transcript/reince-priebus-on-trumps-stance-on...</td>\n",
       "      <td>2016-11-27T11:05:29-05:00</td>\n",
       "      <td>2017-01-23T14:25:28-05:00</td>\n",
       "      <td>{'name': 'FOX NEWS SUNDAY', 'url': '/category/...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32259</th>\n",
       "      <td>https://a57.foxnews.com///static.foxnews.com/s...</td>\n",
       "      <td>Kellyanne Conway talks transition; Exclusive: ...</td>\n",
       "      <td>President-elect Donald Trump's senior adviser ...</td>\n",
       "      <td>/transcript/kellyanne-conway-talks-transition-...</td>\n",
       "      <td>2016-12-04T11:47:40-05:00</td>\n",
       "      <td>2017-01-23T14:22:31-05:00</td>\n",
       "      <td>{'name': 'FOX NEWS SUNDAY', 'url': '/category/...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32260</th>\n",
       "      <td>https://a57.foxnews.com///static.foxnews.com/s...</td>\n",
       "      <td>Exclusive: Donald Trump on Cabinet picks, tran...</td>\n",
       "      <td>President-elect on 'Fox News Sunday' for his f...</td>\n",
       "      <td>/transcript/exclusive-donald-trump-on-cabinet-...</td>\n",
       "      <td>2016-12-11T11:15:23-05:00</td>\n",
       "      <td>2017-01-23T14:19:08-05:00</td>\n",
       "      <td>{'name': 'FOX NEWS SUNDAY', 'url': '/category/...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32261</th>\n",
       "      <td>https://a57.foxnews.com///static.foxnews.com/s...</td>\n",
       "      <td>Reince Priebus on war of words with White Hous...</td>\n",
       "      <td>Incoming White House chief of staff weighs in ...</td>\n",
       "      <td>/transcript/reince-priebus-on-war-of-words-wit...</td>\n",
       "      <td>2016-12-18T11:09:39-05:00</td>\n",
       "      <td>2017-01-23T14:16:31-05:00</td>\n",
       "      <td>{'name': 'FOX NEWS SUNDAY', 'url': '/category/...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32262</th>\n",
       "      <td>https://a57.foxnews.com///static.foxnews.com/s...</td>\n",
       "      <td>Newt Gingrich on national security challenges ...</td>\n",
       "      <td>Former speaker of the House weighs in on \"Fox ...</td>\n",
       "      <td>/transcript/newt-gingrich-on-national-security...</td>\n",
       "      <td>2016-12-25T12:17:31-05:00</td>\n",
       "      <td>2017-01-23T14:14:18-05:00</td>\n",
       "      <td>{'name': 'FOX NEWS SUNDAY', 'url': '/category/...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>32263 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                imageUrl  \\\n",
       "0      https://a57.foxnews.com/cf-images.us-east-1.pr...   \n",
       "1      https://a57.foxnews.com/cf-images.us-east-1.pr...   \n",
       "2      https://a57.foxnews.com/cf-images.us-east-1.pr...   \n",
       "3      https://a57.foxnews.com/cf-images.us-east-1.pr...   \n",
       "4      https://a57.foxnews.com/cf-images.us-east-1.pr...   \n",
       "...                                                  ...   \n",
       "32258  https://a57.foxnews.com///static.foxnews.com/s...   \n",
       "32259  https://a57.foxnews.com///static.foxnews.com/s...   \n",
       "32260  https://a57.foxnews.com///static.foxnews.com/s...   \n",
       "32261  https://a57.foxnews.com///static.foxnews.com/s...   \n",
       "32262  https://a57.foxnews.com///static.foxnews.com/s...   \n",
       "\n",
       "                                                   title  \\\n",
       "0                      'Fox News Sunday' on May 29, 2022   \n",
       "1                      'Fox News Sunday' on May 22, 2022   \n",
       "2                      'Fox News Sunday' on May 15, 2022   \n",
       "3                    ‘Fox News Sunday’ on April 17, 2022   \n",
       "4                    'Fox News Sunday' on April 10, 2022   \n",
       "...                                                  ...   \n",
       "32258  Reince Priebus on Trump's stance on Cuba, Cabi...   \n",
       "32259  Kellyanne Conway talks transition; Exclusive: ...   \n",
       "32260  Exclusive: Donald Trump on Cabinet picks, tran...   \n",
       "32261  Reince Priebus on war of words with White Hous...   \n",
       "32262  Newt Gingrich on national security challenges ...   \n",
       "\n",
       "                                             description  \\\n",
       "0      \"Fox News Sunday\" welcomed guests Mo Brooks, O...   \n",
       "1      \"Fox News Sunday\" welcomed guests Brian Deese,...   \n",
       "2      \"Fox News Sunday\" welcomed guests Kevin Stitt,...   \n",
       "3      \"Fox News Sunday\" welcomed guests Josh Krausha...   \n",
       "4      \"Fox News Sunday\" welcomed guests Mitch McConn...   \n",
       "...                                                  ...   \n",
       "32258  Incoming White House chief of staff weighs in ...   \n",
       "32259  President-elect Donald Trump's senior adviser ...   \n",
       "32260  President-elect on 'Fox News Sunday' for his f...   \n",
       "32261  Incoming White House chief of staff weighs in ...   \n",
       "32262  Former speaker of the House weighs in on \"Fox ...   \n",
       "\n",
       "                                                     url  \\\n",
       "0             /transcript/fox-news-sunday-on-may-29-2022   \n",
       "1             /transcript/fox-news-sunday-on-may-22-2022   \n",
       "2             /transcript/fox-news-sunday-on-may-15-2022   \n",
       "3           /transcript/fox-news-sunday-on-april-17-2022   \n",
       "4      /shows/fox-news-sunday/fox-news-sunday-on-apri...   \n",
       "...                                                  ...   \n",
       "32258  /transcript/reince-priebus-on-trumps-stance-on...   \n",
       "32259  /transcript/kellyanne-conway-talks-transition-...   \n",
       "32260  /transcript/exclusive-donald-trump-on-cabinet-...   \n",
       "32261  /transcript/reince-priebus-on-war-of-words-wit...   \n",
       "32262  /transcript/newt-gingrich-on-national-security...   \n",
       "\n",
       "                 publicationDate          lastPublishedDate  \\\n",
       "0      2022-05-29T14:08:01-04:00  2022-05-29T14:08:01-04:00   \n",
       "1      2022-05-22T13:59:09-04:00  2022-05-22T13:59:09-04:00   \n",
       "2      2022-05-15T13:59:21-04:00  2022-05-15T13:59:21-04:00   \n",
       "3      2022-04-17T14:08:36-04:00  2022-04-17T14:08:36-04:00   \n",
       "4      2022-04-10T13:59:48-04:00  2022-04-10T13:59:48-04:00   \n",
       "...                          ...                        ...   \n",
       "32258  2016-11-27T11:05:29-05:00  2017-01-23T14:25:28-05:00   \n",
       "32259  2016-12-04T11:47:40-05:00  2017-01-23T14:22:31-05:00   \n",
       "32260  2016-12-11T11:15:23-05:00  2017-01-23T14:19:08-05:00   \n",
       "32261  2016-12-18T11:09:39-05:00  2017-01-23T14:16:31-05:00   \n",
       "32262  2016-12-25T12:17:31-05:00  2017-01-23T14:14:18-05:00   \n",
       "\n",
       "                                                category  isBreaking  isLive  \\\n",
       "0      {'name': 'TRANSCRIPT', 'url': '/category/shows...       False   False   \n",
       "1      {'name': 'TRANSCRIPT', 'url': '/category/shows...       False   False   \n",
       "2      {'name': 'TRANSCRIPT', 'url': '/category/shows...       False   False   \n",
       "3      {'name': 'TRANSCRIPT', 'url': '/category/shows...       False   False   \n",
       "4      {'name': 'TRANSCRIPT', 'url': '/category/shows...       False   False   \n",
       "...                                                  ...         ...     ...   \n",
       "32258  {'name': 'FOX NEWS SUNDAY', 'url': '/category/...       False   False   \n",
       "32259  {'name': 'FOX NEWS SUNDAY', 'url': '/category/...       False   False   \n",
       "32260  {'name': 'FOX NEWS SUNDAY', 'url': '/category/...       False   False   \n",
       "32261  {'name': 'FOX NEWS SUNDAY', 'url': '/category/...       False   False   \n",
       "32262  {'name': 'FOX NEWS SUNDAY', 'url': '/category/...       False   False   \n",
       "\n",
       "       duration  \n",
       "0           NaN  \n",
       "1           NaN  \n",
       "2           NaN  \n",
       "3           NaN  \n",
       "4           NaN  \n",
       "...         ...  \n",
       "32258       NaN  \n",
       "32259       NaN  \n",
       "32260       NaN  \n",
       "32261       NaN  \n",
       "32262       NaN  \n",
       "\n",
       "[32263 rows x 10 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('foxnews-transcript-urls.csv.gz')\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "ace6ee44",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>imageUrl</th>\n",
       "      <th>title</th>\n",
       "      <th>description</th>\n",
       "      <th>url</th>\n",
       "      <th>publicationDate</th>\n",
       "      <th>lastPublishedDate</th>\n",
       "      <th>category</th>\n",
       "      <th>isBreaking</th>\n",
       "      <th>isLive</th>\n",
       "      <th>duration</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>https://a57.foxnews.com/cf-images.us-east-1.pr...</td>\n",
       "      <td>'Fox News Sunday' on May 29, 2022</td>\n",
       "      <td>\"Fox News Sunday\" welcomed guests Mo Brooks, O...</td>\n",
       "      <td>/transcript/fox-news-sunday-on-may-29-2022</td>\n",
       "      <td>2022-05-29T14:08:01-04:00</td>\n",
       "      <td>2022-05-29T14:08:01-04:00</td>\n",
       "      <td>{'name': 'TRANSCRIPT', 'url': '/category/shows...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>https://a57.foxnews.com/cf-images.us-east-1.pr...</td>\n",
       "      <td>'Fox News Sunday' on May 22, 2022</td>\n",
       "      <td>\"Fox News Sunday\" welcomed guests Brian Deese,...</td>\n",
       "      <td>/transcript/fox-news-sunday-on-may-22-2022</td>\n",
       "      <td>2022-05-22T13:59:09-04:00</td>\n",
       "      <td>2022-05-22T13:59:09-04:00</td>\n",
       "      <td>{'name': 'TRANSCRIPT', 'url': '/category/shows...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>https://a57.foxnews.com/cf-images.us-east-1.pr...</td>\n",
       "      <td>'Fox News Sunday' on May 15, 2022</td>\n",
       "      <td>\"Fox News Sunday\" welcomed guests Kevin Stitt,...</td>\n",
       "      <td>/transcript/fox-news-sunday-on-may-15-2022</td>\n",
       "      <td>2022-05-15T13:59:21-04:00</td>\n",
       "      <td>2022-05-15T13:59:21-04:00</td>\n",
       "      <td>{'name': 'TRANSCRIPT', 'url': '/category/shows...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>https://a57.foxnews.com/cf-images.us-east-1.pr...</td>\n",
       "      <td>‘Fox News Sunday’ on April 17, 2022</td>\n",
       "      <td>\"Fox News Sunday\" welcomed guests Josh Krausha...</td>\n",
       "      <td>/transcript/fox-news-sunday-on-april-17-2022</td>\n",
       "      <td>2022-04-17T14:08:36-04:00</td>\n",
       "      <td>2022-04-17T14:08:36-04:00</td>\n",
       "      <td>{'name': 'TRANSCRIPT', 'url': '/category/shows...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>https://a57.foxnews.com/cf-images.us-east-1.pr...</td>\n",
       "      <td>'Fox News Sunday' on April 10, 2022</td>\n",
       "      <td>\"Fox News Sunday\" welcomed guests Mitch McConn...</td>\n",
       "      <td>/shows/fox-news-sunday/fox-news-sunday-on-apri...</td>\n",
       "      <td>2022-04-10T13:59:48-04:00</td>\n",
       "      <td>2022-04-10T13:59:48-04:00</td>\n",
       "      <td>{'name': 'TRANSCRIPT', 'url': '/category/shows...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32258</th>\n",
       "      <td>https://a57.foxnews.com///static.foxnews.com/s...</td>\n",
       "      <td>Reince Priebus on Trump's stance on Cuba, Cabi...</td>\n",
       "      <td>Incoming White House chief of staff weighs in ...</td>\n",
       "      <td>/transcript/reince-priebus-on-trumps-stance-on...</td>\n",
       "      <td>2016-11-27T11:05:29-05:00</td>\n",
       "      <td>2017-01-23T14:25:28-05:00</td>\n",
       "      <td>{'name': 'FOX NEWS SUNDAY', 'url': '/category/...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32259</th>\n",
       "      <td>https://a57.foxnews.com///static.foxnews.com/s...</td>\n",
       "      <td>Kellyanne Conway talks transition; Exclusive: ...</td>\n",
       "      <td>President-elect Donald Trump's senior adviser ...</td>\n",
       "      <td>/transcript/kellyanne-conway-talks-transition-...</td>\n",
       "      <td>2016-12-04T11:47:40-05:00</td>\n",
       "      <td>2017-01-23T14:22:31-05:00</td>\n",
       "      <td>{'name': 'FOX NEWS SUNDAY', 'url': '/category/...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32260</th>\n",
       "      <td>https://a57.foxnews.com///static.foxnews.com/s...</td>\n",
       "      <td>Exclusive: Donald Trump on Cabinet picks, tran...</td>\n",
       "      <td>President-elect on 'Fox News Sunday' for his f...</td>\n",
       "      <td>/transcript/exclusive-donald-trump-on-cabinet-...</td>\n",
       "      <td>2016-12-11T11:15:23-05:00</td>\n",
       "      <td>2017-01-23T14:19:08-05:00</td>\n",
       "      <td>{'name': 'FOX NEWS SUNDAY', 'url': '/category/...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32261</th>\n",
       "      <td>https://a57.foxnews.com///static.foxnews.com/s...</td>\n",
       "      <td>Reince Priebus on war of words with White Hous...</td>\n",
       "      <td>Incoming White House chief of staff weighs in ...</td>\n",
       "      <td>/transcript/reince-priebus-on-war-of-words-wit...</td>\n",
       "      <td>2016-12-18T11:09:39-05:00</td>\n",
       "      <td>2017-01-23T14:16:31-05:00</td>\n",
       "      <td>{'name': 'FOX NEWS SUNDAY', 'url': '/category/...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32262</th>\n",
       "      <td>https://a57.foxnews.com///static.foxnews.com/s...</td>\n",
       "      <td>Newt Gingrich on national security challenges ...</td>\n",
       "      <td>Former speaker of the House weighs in on \"Fox ...</td>\n",
       "      <td>/transcript/newt-gingrich-on-national-security...</td>\n",
       "      <td>2016-12-25T12:17:31-05:00</td>\n",
       "      <td>2017-01-23T14:14:18-05:00</td>\n",
       "      <td>{'name': 'FOX NEWS SUNDAY', 'url': '/category/...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>24530 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                imageUrl  \\\n",
       "0      https://a57.foxnews.com/cf-images.us-east-1.pr...   \n",
       "1      https://a57.foxnews.com/cf-images.us-east-1.pr...   \n",
       "2      https://a57.foxnews.com/cf-images.us-east-1.pr...   \n",
       "3      https://a57.foxnews.com/cf-images.us-east-1.pr...   \n",
       "4      https://a57.foxnews.com/cf-images.us-east-1.pr...   \n",
       "...                                                  ...   \n",
       "32258  https://a57.foxnews.com///static.foxnews.com/s...   \n",
       "32259  https://a57.foxnews.com///static.foxnews.com/s...   \n",
       "32260  https://a57.foxnews.com///static.foxnews.com/s...   \n",
       "32261  https://a57.foxnews.com///static.foxnews.com/s...   \n",
       "32262  https://a57.foxnews.com///static.foxnews.com/s...   \n",
       "\n",
       "                                                   title  \\\n",
       "0                      'Fox News Sunday' on May 29, 2022   \n",
       "1                      'Fox News Sunday' on May 22, 2022   \n",
       "2                      'Fox News Sunday' on May 15, 2022   \n",
       "3                    ‘Fox News Sunday’ on April 17, 2022   \n",
       "4                    'Fox News Sunday' on April 10, 2022   \n",
       "...                                                  ...   \n",
       "32258  Reince Priebus on Trump's stance on Cuba, Cabi...   \n",
       "32259  Kellyanne Conway talks transition; Exclusive: ...   \n",
       "32260  Exclusive: Donald Trump on Cabinet picks, tran...   \n",
       "32261  Reince Priebus on war of words with White Hous...   \n",
       "32262  Newt Gingrich on national security challenges ...   \n",
       "\n",
       "                                             description  \\\n",
       "0      \"Fox News Sunday\" welcomed guests Mo Brooks, O...   \n",
       "1      \"Fox News Sunday\" welcomed guests Brian Deese,...   \n",
       "2      \"Fox News Sunday\" welcomed guests Kevin Stitt,...   \n",
       "3      \"Fox News Sunday\" welcomed guests Josh Krausha...   \n",
       "4      \"Fox News Sunday\" welcomed guests Mitch McConn...   \n",
       "...                                                  ...   \n",
       "32258  Incoming White House chief of staff weighs in ...   \n",
       "32259  President-elect Donald Trump's senior adviser ...   \n",
       "32260  President-elect on 'Fox News Sunday' for his f...   \n",
       "32261  Incoming White House chief of staff weighs in ...   \n",
       "32262  Former speaker of the House weighs in on \"Fox ...   \n",
       "\n",
       "                                                     url  \\\n",
       "0             /transcript/fox-news-sunday-on-may-29-2022   \n",
       "1             /transcript/fox-news-sunday-on-may-22-2022   \n",
       "2             /transcript/fox-news-sunday-on-may-15-2022   \n",
       "3           /transcript/fox-news-sunday-on-april-17-2022   \n",
       "4      /shows/fox-news-sunday/fox-news-sunday-on-apri...   \n",
       "...                                                  ...   \n",
       "32258  /transcript/reince-priebus-on-trumps-stance-on...   \n",
       "32259  /transcript/kellyanne-conway-talks-transition-...   \n",
       "32260  /transcript/exclusive-donald-trump-on-cabinet-...   \n",
       "32261  /transcript/reince-priebus-on-war-of-words-wit...   \n",
       "32262  /transcript/newt-gingrich-on-national-security...   \n",
       "\n",
       "                 publicationDate          lastPublishedDate  \\\n",
       "0      2022-05-29T14:08:01-04:00  2022-05-29T14:08:01-04:00   \n",
       "1      2022-05-22T13:59:09-04:00  2022-05-22T13:59:09-04:00   \n",
       "2      2022-05-15T13:59:21-04:00  2022-05-15T13:59:21-04:00   \n",
       "3      2022-04-17T14:08:36-04:00  2022-04-17T14:08:36-04:00   \n",
       "4      2022-04-10T13:59:48-04:00  2022-04-10T13:59:48-04:00   \n",
       "...                          ...                        ...   \n",
       "32258  2016-11-27T11:05:29-05:00  2017-01-23T14:25:28-05:00   \n",
       "32259  2016-12-04T11:47:40-05:00  2017-01-23T14:22:31-05:00   \n",
       "32260  2016-12-11T11:15:23-05:00  2017-01-23T14:19:08-05:00   \n",
       "32261  2016-12-18T11:09:39-05:00  2017-01-23T14:16:31-05:00   \n",
       "32262  2016-12-25T12:17:31-05:00  2017-01-23T14:14:18-05:00   \n",
       "\n",
       "                                                category  isBreaking  isLive  \\\n",
       "0      {'name': 'TRANSCRIPT', 'url': '/category/shows...       False   False   \n",
       "1      {'name': 'TRANSCRIPT', 'url': '/category/shows...       False   False   \n",
       "2      {'name': 'TRANSCRIPT', 'url': '/category/shows...       False   False   \n",
       "3      {'name': 'TRANSCRIPT', 'url': '/category/shows...       False   False   \n",
       "4      {'name': 'TRANSCRIPT', 'url': '/category/shows...       False   False   \n",
       "...                                                  ...         ...     ...   \n",
       "32258  {'name': 'FOX NEWS SUNDAY', 'url': '/category/...       False   False   \n",
       "32259  {'name': 'FOX NEWS SUNDAY', 'url': '/category/...       False   False   \n",
       "32260  {'name': 'FOX NEWS SUNDAY', 'url': '/category/...       False   False   \n",
       "32261  {'name': 'FOX NEWS SUNDAY', 'url': '/category/...       False   False   \n",
       "32262  {'name': 'FOX NEWS SUNDAY', 'url': '/category/...       False   False   \n",
       "\n",
       "       duration  \n",
       "0           NaN  \n",
       "1           NaN  \n",
       "2           NaN  \n",
       "3           NaN  \n",
       "4           NaN  \n",
       "...         ...  \n",
       "32258       NaN  \n",
       "32259       NaN  \n",
       "32260       NaN  \n",
       "32261       NaN  \n",
       "32262       NaN  \n",
       "\n",
       "[24530 rows x 10 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.drop_duplicates(subset=['url'], inplace=True)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "87539df4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "24530"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df.url.unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "eca7990e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>imageUrl</th>\n",
       "      <th>title</th>\n",
       "      <th>description</th>\n",
       "      <th>url</th>\n",
       "      <th>publicationDate</th>\n",
       "      <th>lastPublishedDate</th>\n",
       "      <th>category</th>\n",
       "      <th>isBreaking</th>\n",
       "      <th>isLive</th>\n",
       "      <th>duration</th>\n",
       "      <th>html_file</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>https://a57.foxnews.com/cf-images.us-east-1.pr...</td>\n",
       "      <td>'Fox News Sunday' on May 29, 2022</td>\n",
       "      <td>\"Fox News Sunday\" welcomed guests Mo Brooks, O...</td>\n",
       "      <td>/transcript/fox-news-sunday-on-may-29-2022</td>\n",
       "      <td>2022-05-29T14:08:01-04:00</td>\n",
       "      <td>2022-05-29T14:08:01-04:00</td>\n",
       "      <td>{'name': 'TRANSCRIPT', 'url': '/category/shows...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>fox-news-sunday-on-may-29-2022.html</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>https://a57.foxnews.com/cf-images.us-east-1.pr...</td>\n",
       "      <td>'Fox News Sunday' on May 22, 2022</td>\n",
       "      <td>\"Fox News Sunday\" welcomed guests Brian Deese,...</td>\n",
       "      <td>/transcript/fox-news-sunday-on-may-22-2022</td>\n",
       "      <td>2022-05-22T13:59:09-04:00</td>\n",
       "      <td>2022-05-22T13:59:09-04:00</td>\n",
       "      <td>{'name': 'TRANSCRIPT', 'url': '/category/shows...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>fox-news-sunday-on-may-22-2022.html</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>https://a57.foxnews.com/cf-images.us-east-1.pr...</td>\n",
       "      <td>'Fox News Sunday' on May 15, 2022</td>\n",
       "      <td>\"Fox News Sunday\" welcomed guests Kevin Stitt,...</td>\n",
       "      <td>/transcript/fox-news-sunday-on-may-15-2022</td>\n",
       "      <td>2022-05-15T13:59:21-04:00</td>\n",
       "      <td>2022-05-15T13:59:21-04:00</td>\n",
       "      <td>{'name': 'TRANSCRIPT', 'url': '/category/shows...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>fox-news-sunday-on-may-15-2022.html</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>https://a57.foxnews.com/cf-images.us-east-1.pr...</td>\n",
       "      <td>‘Fox News Sunday’ on April 17, 2022</td>\n",
       "      <td>\"Fox News Sunday\" welcomed guests Josh Krausha...</td>\n",
       "      <td>/transcript/fox-news-sunday-on-april-17-2022</td>\n",
       "      <td>2022-04-17T14:08:36-04:00</td>\n",
       "      <td>2022-04-17T14:08:36-04:00</td>\n",
       "      <td>{'name': 'TRANSCRIPT', 'url': '/category/shows...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>fox-news-sunday-on-april-17-2022.html</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>https://a57.foxnews.com/cf-images.us-east-1.pr...</td>\n",
       "      <td>'Fox News Sunday' on April 10, 2022</td>\n",
       "      <td>\"Fox News Sunday\" welcomed guests Mitch McConn...</td>\n",
       "      <td>/shows/fox-news-sunday/fox-news-sunday-on-apri...</td>\n",
       "      <td>2022-04-10T13:59:48-04:00</td>\n",
       "      <td>2022-04-10T13:59:48-04:00</td>\n",
       "      <td>{'name': 'TRANSCRIPT', 'url': '/category/shows...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>fox-news-sunday-on-april-10-2022.html</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24525</th>\n",
       "      <td>https://a57.foxnews.com///static.foxnews.com/s...</td>\n",
       "      <td>Reince Priebus on Trump's stance on Cuba, Cabi...</td>\n",
       "      <td>Incoming White House chief of staff weighs in ...</td>\n",
       "      <td>/transcript/reince-priebus-on-trumps-stance-on...</td>\n",
       "      <td>2016-11-27T11:05:29-05:00</td>\n",
       "      <td>2017-01-23T14:25:28-05:00</td>\n",
       "      <td>{'name': 'FOX NEWS SUNDAY', 'url': '/category/...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>reince-priebus-on-trumps-stance-on-cuba-cabine...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24526</th>\n",
       "      <td>https://a57.foxnews.com///static.foxnews.com/s...</td>\n",
       "      <td>Kellyanne Conway talks transition; Exclusive: ...</td>\n",
       "      <td>President-elect Donald Trump's senior adviser ...</td>\n",
       "      <td>/transcript/kellyanne-conway-talks-transition-...</td>\n",
       "      <td>2016-12-04T11:47:40-05:00</td>\n",
       "      <td>2017-01-23T14:22:31-05:00</td>\n",
       "      <td>{'name': 'FOX NEWS SUNDAY', 'url': '/category/...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>kellyanne-conway-talks-transition-exclusive-ji...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24527</th>\n",
       "      <td>https://a57.foxnews.com///static.foxnews.com/s...</td>\n",
       "      <td>Exclusive: Donald Trump on Cabinet picks, tran...</td>\n",
       "      <td>President-elect on 'Fox News Sunday' for his f...</td>\n",
       "      <td>/transcript/exclusive-donald-trump-on-cabinet-...</td>\n",
       "      <td>2016-12-11T11:15:23-05:00</td>\n",
       "      <td>2017-01-23T14:19:08-05:00</td>\n",
       "      <td>{'name': 'FOX NEWS SUNDAY', 'url': '/category/...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>exclusive-donald-trump-on-cabinet-picks-transi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24528</th>\n",
       "      <td>https://a57.foxnews.com///static.foxnews.com/s...</td>\n",
       "      <td>Reince Priebus on war of words with White Hous...</td>\n",
       "      <td>Incoming White House chief of staff weighs in ...</td>\n",
       "      <td>/transcript/reince-priebus-on-war-of-words-wit...</td>\n",
       "      <td>2016-12-18T11:09:39-05:00</td>\n",
       "      <td>2017-01-23T14:16:31-05:00</td>\n",
       "      <td>{'name': 'FOX NEWS SUNDAY', 'url': '/category/...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>reince-priebus-on-war-of-words-with-white-hous...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24529</th>\n",
       "      <td>https://a57.foxnews.com///static.foxnews.com/s...</td>\n",
       "      <td>Newt Gingrich on national security challenges ...</td>\n",
       "      <td>Former speaker of the House weighs in on \"Fox ...</td>\n",
       "      <td>/transcript/newt-gingrich-on-national-security...</td>\n",
       "      <td>2016-12-25T12:17:31-05:00</td>\n",
       "      <td>2017-01-23T14:14:18-05:00</td>\n",
       "      <td>{'name': 'FOX NEWS SUNDAY', 'url': '/category/...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>newt-gingrich-on-national-security-challenges-...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>24530 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                imageUrl  \\\n",
       "0      https://a57.foxnews.com/cf-images.us-east-1.pr...   \n",
       "1      https://a57.foxnews.com/cf-images.us-east-1.pr...   \n",
       "2      https://a57.foxnews.com/cf-images.us-east-1.pr...   \n",
       "3      https://a57.foxnews.com/cf-images.us-east-1.pr...   \n",
       "4      https://a57.foxnews.com/cf-images.us-east-1.pr...   \n",
       "...                                                  ...   \n",
       "24525  https://a57.foxnews.com///static.foxnews.com/s...   \n",
       "24526  https://a57.foxnews.com///static.foxnews.com/s...   \n",
       "24527  https://a57.foxnews.com///static.foxnews.com/s...   \n",
       "24528  https://a57.foxnews.com///static.foxnews.com/s...   \n",
       "24529  https://a57.foxnews.com///static.foxnews.com/s...   \n",
       "\n",
       "                                                   title  \\\n",
       "0                      'Fox News Sunday' on May 29, 2022   \n",
       "1                      'Fox News Sunday' on May 22, 2022   \n",
       "2                      'Fox News Sunday' on May 15, 2022   \n",
       "3                    ‘Fox News Sunday’ on April 17, 2022   \n",
       "4                    'Fox News Sunday' on April 10, 2022   \n",
       "...                                                  ...   \n",
       "24525  Reince Priebus on Trump's stance on Cuba, Cabi...   \n",
       "24526  Kellyanne Conway talks transition; Exclusive: ...   \n",
       "24527  Exclusive: Donald Trump on Cabinet picks, tran...   \n",
       "24528  Reince Priebus on war of words with White Hous...   \n",
       "24529  Newt Gingrich on national security challenges ...   \n",
       "\n",
       "                                             description  \\\n",
       "0      \"Fox News Sunday\" welcomed guests Mo Brooks, O...   \n",
       "1      \"Fox News Sunday\" welcomed guests Brian Deese,...   \n",
       "2      \"Fox News Sunday\" welcomed guests Kevin Stitt,...   \n",
       "3      \"Fox News Sunday\" welcomed guests Josh Krausha...   \n",
       "4      \"Fox News Sunday\" welcomed guests Mitch McConn...   \n",
       "...                                                  ...   \n",
       "24525  Incoming White House chief of staff weighs in ...   \n",
       "24526  President-elect Donald Trump's senior adviser ...   \n",
       "24527  President-elect on 'Fox News Sunday' for his f...   \n",
       "24528  Incoming White House chief of staff weighs in ...   \n",
       "24529  Former speaker of the House weighs in on \"Fox ...   \n",
       "\n",
       "                                                     url  \\\n",
       "0             /transcript/fox-news-sunday-on-may-29-2022   \n",
       "1             /transcript/fox-news-sunday-on-may-22-2022   \n",
       "2             /transcript/fox-news-sunday-on-may-15-2022   \n",
       "3           /transcript/fox-news-sunday-on-april-17-2022   \n",
       "4      /shows/fox-news-sunday/fox-news-sunday-on-apri...   \n",
       "...                                                  ...   \n",
       "24525  /transcript/reince-priebus-on-trumps-stance-on...   \n",
       "24526  /transcript/kellyanne-conway-talks-transition-...   \n",
       "24527  /transcript/exclusive-donald-trump-on-cabinet-...   \n",
       "24528  /transcript/reince-priebus-on-war-of-words-wit...   \n",
       "24529  /transcript/newt-gingrich-on-national-security...   \n",
       "\n",
       "                 publicationDate          lastPublishedDate  \\\n",
       "0      2022-05-29T14:08:01-04:00  2022-05-29T14:08:01-04:00   \n",
       "1      2022-05-22T13:59:09-04:00  2022-05-22T13:59:09-04:00   \n",
       "2      2022-05-15T13:59:21-04:00  2022-05-15T13:59:21-04:00   \n",
       "3      2022-04-17T14:08:36-04:00  2022-04-17T14:08:36-04:00   \n",
       "4      2022-04-10T13:59:48-04:00  2022-04-10T13:59:48-04:00   \n",
       "...                          ...                        ...   \n",
       "24525  2016-11-27T11:05:29-05:00  2017-01-23T14:25:28-05:00   \n",
       "24526  2016-12-04T11:47:40-05:00  2017-01-23T14:22:31-05:00   \n",
       "24527  2016-12-11T11:15:23-05:00  2017-01-23T14:19:08-05:00   \n",
       "24528  2016-12-18T11:09:39-05:00  2017-01-23T14:16:31-05:00   \n",
       "24529  2016-12-25T12:17:31-05:00  2017-01-23T14:14:18-05:00   \n",
       "\n",
       "                                                category  isBreaking  isLive  \\\n",
       "0      {'name': 'TRANSCRIPT', 'url': '/category/shows...       False   False   \n",
       "1      {'name': 'TRANSCRIPT', 'url': '/category/shows...       False   False   \n",
       "2      {'name': 'TRANSCRIPT', 'url': '/category/shows...       False   False   \n",
       "3      {'name': 'TRANSCRIPT', 'url': '/category/shows...       False   False   \n",
       "4      {'name': 'TRANSCRIPT', 'url': '/category/shows...       False   False   \n",
       "...                                                  ...         ...     ...   \n",
       "24525  {'name': 'FOX NEWS SUNDAY', 'url': '/category/...       False   False   \n",
       "24526  {'name': 'FOX NEWS SUNDAY', 'url': '/category/...       False   False   \n",
       "24527  {'name': 'FOX NEWS SUNDAY', 'url': '/category/...       False   False   \n",
       "24528  {'name': 'FOX NEWS SUNDAY', 'url': '/category/...       False   False   \n",
       "24529  {'name': 'FOX NEWS SUNDAY', 'url': '/category/...       False   False   \n",
       "\n",
       "       duration                                          html_file  \n",
       "0           NaN                fox-news-sunday-on-may-29-2022.html  \n",
       "1           NaN                fox-news-sunday-on-may-22-2022.html  \n",
       "2           NaN                fox-news-sunday-on-may-15-2022.html  \n",
       "3           NaN              fox-news-sunday-on-april-17-2022.html  \n",
       "4           NaN              fox-news-sunday-on-april-10-2022.html  \n",
       "...         ...                                                ...  \n",
       "24525       NaN  reince-priebus-on-trumps-stance-on-cuba-cabine...  \n",
       "24526       NaN  kellyanne-conway-talks-transition-exclusive-ji...  \n",
       "24527       NaN  exclusive-donald-trump-on-cabinet-picks-transi...  \n",
       "24528       NaN  reince-priebus-on-war-of-words-with-white-hous...  \n",
       "24529       NaN  newt-gingrich-on-national-security-challenges-...  \n",
       "\n",
       "[24530 rows x 11 columns]"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['html_file'] = df.url.apply(lambda c: c.split('/')[-1] + '.html')\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "84c39b97",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>imageUrl</th>\n",
       "      <th>title</th>\n",
       "      <th>description</th>\n",
       "      <th>url</th>\n",
       "      <th>publicationDate</th>\n",
       "      <th>lastPublishedDate</th>\n",
       "      <th>category</th>\n",
       "      <th>isBreaking</th>\n",
       "      <th>isLive</th>\n",
       "      <th>duration</th>\n",
       "      <th>html_file</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2650</th>\n",
       "      <td>https://a57.foxnews.com/static.foxnews.com/fox...</td>\n",
       "      <td>Tucker Carlson: Biden’s economic policy, labor...</td>\n",
       "      <td>'Tucker Carlson Tonight' host Tucker Carlson t...</td>\n",
       "      <td>/media/tucker-carlson-bidens-economic-policy</td>\n",
       "      <td>2021-12-17T01:14:56-05:00</td>\n",
       "      <td>2021-12-17T01:14:56-05:00</td>\n",
       "      <td>{'name': 'OPINION', 'url': '/category/opinion'}</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>tucker-carlson-bidens-economic-policy.html</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               imageUrl  \\\n",
       "2650  https://a57.foxnews.com/static.foxnews.com/fox...   \n",
       "\n",
       "                                                  title  \\\n",
       "2650  Tucker Carlson: Biden’s economic policy, labor...   \n",
       "\n",
       "                                            description  \\\n",
       "2650  'Tucker Carlson Tonight' host Tucker Carlson t...   \n",
       "\n",
       "                                               url            publicationDate  \\\n",
       "2650  /media/tucker-carlson-bidens-economic-policy  2021-12-17T01:14:56-05:00   \n",
       "\n",
       "              lastPublishedDate  \\\n",
       "2650  2021-12-17T01:14:56-05:00   \n",
       "\n",
       "                                             category  isBreaking  isLive  \\\n",
       "2650  {'name': 'OPINION', 'url': '/category/opinion'}       False   False   \n",
       "\n",
       "      duration                                   html_file  \n",
       "2650       NaN  tucker-carlson-bidens-economic-policy.html  "
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df.duplicated(subset=['html_file'])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "fea85bab",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>imageUrl</th>\n",
       "      <th>title</th>\n",
       "      <th>description</th>\n",
       "      <th>url</th>\n",
       "      <th>publicationDate</th>\n",
       "      <th>lastPublishedDate</th>\n",
       "      <th>category</th>\n",
       "      <th>isBreaking</th>\n",
       "      <th>isLive</th>\n",
       "      <th>duration</th>\n",
       "      <th>html_file</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>https://a57.foxnews.com/cf-images.us-east-1.pr...</td>\n",
       "      <td>'Fox News Sunday' on May 29, 2022</td>\n",
       "      <td>\"Fox News Sunday\" welcomed guests Mo Brooks, O...</td>\n",
       "      <td>/transcript/fox-news-sunday-on-may-29-2022</td>\n",
       "      <td>2022-05-29T14:08:01-04:00</td>\n",
       "      <td>2022-05-29T14:08:01-04:00</td>\n",
       "      <td>{'name': 'TRANSCRIPT', 'url': '/category/shows...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>fox-news-sunday-on-may-29-2022.html</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>https://a57.foxnews.com/cf-images.us-east-1.pr...</td>\n",
       "      <td>'Fox News Sunday' on May 22, 2022</td>\n",
       "      <td>\"Fox News Sunday\" welcomed guests Brian Deese,...</td>\n",
       "      <td>/transcript/fox-news-sunday-on-may-22-2022</td>\n",
       "      <td>2022-05-22T13:59:09-04:00</td>\n",
       "      <td>2022-05-22T13:59:09-04:00</td>\n",
       "      <td>{'name': 'TRANSCRIPT', 'url': '/category/shows...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>fox-news-sunday-on-may-22-2022.html</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>https://a57.foxnews.com/cf-images.us-east-1.pr...</td>\n",
       "      <td>'Fox News Sunday' on May 15, 2022</td>\n",
       "      <td>\"Fox News Sunday\" welcomed guests Kevin Stitt,...</td>\n",
       "      <td>/transcript/fox-news-sunday-on-may-15-2022</td>\n",
       "      <td>2022-05-15T13:59:21-04:00</td>\n",
       "      <td>2022-05-15T13:59:21-04:00</td>\n",
       "      <td>{'name': 'TRANSCRIPT', 'url': '/category/shows...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>fox-news-sunday-on-may-15-2022.html</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>https://a57.foxnews.com/cf-images.us-east-1.pr...</td>\n",
       "      <td>‘Fox News Sunday’ on April 17, 2022</td>\n",
       "      <td>\"Fox News Sunday\" welcomed guests Josh Krausha...</td>\n",
       "      <td>/transcript/fox-news-sunday-on-april-17-2022</td>\n",
       "      <td>2022-04-17T14:08:36-04:00</td>\n",
       "      <td>2022-04-17T14:08:36-04:00</td>\n",
       "      <td>{'name': 'TRANSCRIPT', 'url': '/category/shows...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>fox-news-sunday-on-april-17-2022.html</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>https://a57.foxnews.com/cf-images.us-east-1.pr...</td>\n",
       "      <td>'Fox News Sunday' on April 10, 2022</td>\n",
       "      <td>\"Fox News Sunday\" welcomed guests Mitch McConn...</td>\n",
       "      <td>/shows/fox-news-sunday/fox-news-sunday-on-apri...</td>\n",
       "      <td>2022-04-10T13:59:48-04:00</td>\n",
       "      <td>2022-04-10T13:59:48-04:00</td>\n",
       "      <td>{'name': 'TRANSCRIPT', 'url': '/category/shows...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>fox-news-sunday-on-april-10-2022.html</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24525</th>\n",
       "      <td>https://a57.foxnews.com///static.foxnews.com/s...</td>\n",
       "      <td>Reince Priebus on Trump's stance on Cuba, Cabi...</td>\n",
       "      <td>Incoming White House chief of staff weighs in ...</td>\n",
       "      <td>/transcript/reince-priebus-on-trumps-stance-on...</td>\n",
       "      <td>2016-11-27T11:05:29-05:00</td>\n",
       "      <td>2017-01-23T14:25:28-05:00</td>\n",
       "      <td>{'name': 'FOX NEWS SUNDAY', 'url': '/category/...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>reince-priebus-on-trumps-stance-on-cuba-cabine...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24526</th>\n",
       "      <td>https://a57.foxnews.com///static.foxnews.com/s...</td>\n",
       "      <td>Kellyanne Conway talks transition; Exclusive: ...</td>\n",
       "      <td>President-elect Donald Trump's senior adviser ...</td>\n",
       "      <td>/transcript/kellyanne-conway-talks-transition-...</td>\n",
       "      <td>2016-12-04T11:47:40-05:00</td>\n",
       "      <td>2017-01-23T14:22:31-05:00</td>\n",
       "      <td>{'name': 'FOX NEWS SUNDAY', 'url': '/category/...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>kellyanne-conway-talks-transition-exclusive-ji...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24527</th>\n",
       "      <td>https://a57.foxnews.com///static.foxnews.com/s...</td>\n",
       "      <td>Exclusive: Donald Trump on Cabinet picks, tran...</td>\n",
       "      <td>President-elect on 'Fox News Sunday' for his f...</td>\n",
       "      <td>/transcript/exclusive-donald-trump-on-cabinet-...</td>\n",
       "      <td>2016-12-11T11:15:23-05:00</td>\n",
       "      <td>2017-01-23T14:19:08-05:00</td>\n",
       "      <td>{'name': 'FOX NEWS SUNDAY', 'url': '/category/...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>exclusive-donald-trump-on-cabinet-picks-transi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24528</th>\n",
       "      <td>https://a57.foxnews.com///static.foxnews.com/s...</td>\n",
       "      <td>Reince Priebus on war of words with White Hous...</td>\n",
       "      <td>Incoming White House chief of staff weighs in ...</td>\n",
       "      <td>/transcript/reince-priebus-on-war-of-words-wit...</td>\n",
       "      <td>2016-12-18T11:09:39-05:00</td>\n",
       "      <td>2017-01-23T14:16:31-05:00</td>\n",
       "      <td>{'name': 'FOX NEWS SUNDAY', 'url': '/category/...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>reince-priebus-on-war-of-words-with-white-hous...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24529</th>\n",
       "      <td>https://a57.foxnews.com///static.foxnews.com/s...</td>\n",
       "      <td>Newt Gingrich on national security challenges ...</td>\n",
       "      <td>Former speaker of the House weighs in on \"Fox ...</td>\n",
       "      <td>/transcript/newt-gingrich-on-national-security...</td>\n",
       "      <td>2016-12-25T12:17:31-05:00</td>\n",
       "      <td>2017-01-23T14:14:18-05:00</td>\n",
       "      <td>{'name': 'FOX NEWS SUNDAY', 'url': '/category/...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>newt-gingrich-on-national-security-challenges-...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>24530 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                imageUrl  \\\n",
       "0      https://a57.foxnews.com/cf-images.us-east-1.pr...   \n",
       "1      https://a57.foxnews.com/cf-images.us-east-1.pr...   \n",
       "2      https://a57.foxnews.com/cf-images.us-east-1.pr...   \n",
       "3      https://a57.foxnews.com/cf-images.us-east-1.pr...   \n",
       "4      https://a57.foxnews.com/cf-images.us-east-1.pr...   \n",
       "...                                                  ...   \n",
       "24525  https://a57.foxnews.com///static.foxnews.com/s...   \n",
       "24526  https://a57.foxnews.com///static.foxnews.com/s...   \n",
       "24527  https://a57.foxnews.com///static.foxnews.com/s...   \n",
       "24528  https://a57.foxnews.com///static.foxnews.com/s...   \n",
       "24529  https://a57.foxnews.com///static.foxnews.com/s...   \n",
       "\n",
       "                                                   title  \\\n",
       "0                      'Fox News Sunday' on May 29, 2022   \n",
       "1                      'Fox News Sunday' on May 22, 2022   \n",
       "2                      'Fox News Sunday' on May 15, 2022   \n",
       "3                    ‘Fox News Sunday’ on April 17, 2022   \n",
       "4                    'Fox News Sunday' on April 10, 2022   \n",
       "...                                                  ...   \n",
       "24525  Reince Priebus on Trump's stance on Cuba, Cabi...   \n",
       "24526  Kellyanne Conway talks transition; Exclusive: ...   \n",
       "24527  Exclusive: Donald Trump on Cabinet picks, tran...   \n",
       "24528  Reince Priebus on war of words with White Hous...   \n",
       "24529  Newt Gingrich on national security challenges ...   \n",
       "\n",
       "                                             description  \\\n",
       "0      \"Fox News Sunday\" welcomed guests Mo Brooks, O...   \n",
       "1      \"Fox News Sunday\" welcomed guests Brian Deese,...   \n",
       "2      \"Fox News Sunday\" welcomed guests Kevin Stitt,...   \n",
       "3      \"Fox News Sunday\" welcomed guests Josh Krausha...   \n",
       "4      \"Fox News Sunday\" welcomed guests Mitch McConn...   \n",
       "...                                                  ...   \n",
       "24525  Incoming White House chief of staff weighs in ...   \n",
       "24526  President-elect Donald Trump's senior adviser ...   \n",
       "24527  President-elect on 'Fox News Sunday' for his f...   \n",
       "24528  Incoming White House chief of staff weighs in ...   \n",
       "24529  Former speaker of the House weighs in on \"Fox ...   \n",
       "\n",
       "                                                     url  \\\n",
       "0             /transcript/fox-news-sunday-on-may-29-2022   \n",
       "1             /transcript/fox-news-sunday-on-may-22-2022   \n",
       "2             /transcript/fox-news-sunday-on-may-15-2022   \n",
       "3           /transcript/fox-news-sunday-on-april-17-2022   \n",
       "4      /shows/fox-news-sunday/fox-news-sunday-on-apri...   \n",
       "...                                                  ...   \n",
       "24525  /transcript/reince-priebus-on-trumps-stance-on...   \n",
       "24526  /transcript/kellyanne-conway-talks-transition-...   \n",
       "24527  /transcript/exclusive-donald-trump-on-cabinet-...   \n",
       "24528  /transcript/reince-priebus-on-war-of-words-wit...   \n",
       "24529  /transcript/newt-gingrich-on-national-security...   \n",
       "\n",
       "                 publicationDate          lastPublishedDate  \\\n",
       "0      2022-05-29T14:08:01-04:00  2022-05-29T14:08:01-04:00   \n",
       "1      2022-05-22T13:59:09-04:00  2022-05-22T13:59:09-04:00   \n",
       "2      2022-05-15T13:59:21-04:00  2022-05-15T13:59:21-04:00   \n",
       "3      2022-04-17T14:08:36-04:00  2022-04-17T14:08:36-04:00   \n",
       "4      2022-04-10T13:59:48-04:00  2022-04-10T13:59:48-04:00   \n",
       "...                          ...                        ...   \n",
       "24525  2016-11-27T11:05:29-05:00  2017-01-23T14:25:28-05:00   \n",
       "24526  2016-12-04T11:47:40-05:00  2017-01-23T14:22:31-05:00   \n",
       "24527  2016-12-11T11:15:23-05:00  2017-01-23T14:19:08-05:00   \n",
       "24528  2016-12-18T11:09:39-05:00  2017-01-23T14:16:31-05:00   \n",
       "24529  2016-12-25T12:17:31-05:00  2017-01-23T14:14:18-05:00   \n",
       "\n",
       "                                                category  isBreaking  isLive  \\\n",
       "0      {'name': 'TRANSCRIPT', 'url': '/category/shows...       False   False   \n",
       "1      {'name': 'TRANSCRIPT', 'url': '/category/shows...       False   False   \n",
       "2      {'name': 'TRANSCRIPT', 'url': '/category/shows...       False   False   \n",
       "3      {'name': 'TRANSCRIPT', 'url': '/category/shows...       False   False   \n",
       "4      {'name': 'TRANSCRIPT', 'url': '/category/shows...       False   False   \n",
       "...                                                  ...         ...     ...   \n",
       "24525  {'name': 'FOX NEWS SUNDAY', 'url': '/category/...       False   False   \n",
       "24526  {'name': 'FOX NEWS SUNDAY', 'url': '/category/...       False   False   \n",
       "24527  {'name': 'FOX NEWS SUNDAY', 'url': '/category/...       False   False   \n",
       "24528  {'name': 'FOX NEWS SUNDAY', 'url': '/category/...       False   False   \n",
       "24529  {'name': 'FOX NEWS SUNDAY', 'url': '/category/...       False   False   \n",
       "\n",
       "       duration                                          html_file  \n",
       "0           NaN                fox-news-sunday-on-may-29-2022.html  \n",
       "1           NaN                fox-news-sunday-on-may-22-2022.html  \n",
       "2           NaN                fox-news-sunday-on-may-15-2022.html  \n",
       "3           NaN              fox-news-sunday-on-april-17-2022.html  \n",
       "4           NaN              fox-news-sunday-on-april-10-2022.html  \n",
       "...         ...                                                ...  \n",
       "24525       NaN  reince-priebus-on-trumps-stance-on-cuba-cabine...  \n",
       "24526       NaN  kellyanne-conway-talks-transition-exclusive-ji...  \n",
       "24527       NaN  exclusive-donald-trump-on-cabinet-picks-transi...  \n",
       "24528       NaN  reince-priebus-on-war-of-words-with-white-hous...  \n",
       "24529       NaN  newt-gingrich-on-national-security-challenges-...  \n",
       "\n",
       "[24530 rows x 11 columns]"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.reset_index(drop=True, inplace=True)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "f9ef8060",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 /transcript/fox-news-sunday-on-may-29-2022\n",
      "100 /transcript/dr-anthony-fauci-on-efforts-to-slow-the-spread-of-coronavirus-in-the-us\n",
      "200 /transcript/rep-trey-gowdy-mueller-probe-should-not-be-shut-down\n",
      "300 /media/ingraham-comments-on-joe-biden-disgustingly-washington-response-question-price-gas-vindictive\n",
      "400 /transcript/ingraham-angle-on-psaki-celebrating-economic-crises\n",
      "500 /transcript/the-ingraham-angle-on-bidens-distraction-tour-critical-race-theory\n",
      "600 /transcript/donald-trump-jr-reacts-to-final-night-of-gop-convention-protesters-outside-the-white-house\n",
      "700 /transcript/uproar-on-the-left-over-sean-spicers-dancing-with-the-stars-success\n",
      "800 /transcript/house-democrats-spend-an-entire-day-reading-the-mueller-report-aloud\n",
      "900 /opinion/laura-ingraham-the-border-wall-becomes-a-border-stall\n",
      "1000 /transcript/trump-on-synagogue-attack-nationalism-and-media-bias\n",
      "1100 /transcript/texas-lieutenant-governor-on-preventing-school-shootings\n",
      "ERROR: 404\n",
      "1200 /transcript/rep-gaetz-speaks-out-about-investigation-double-standards\n",
      "1300 /transcript/hannity-on-bidens-press-conference\n",
      "1400 /transcript/hannity-on-milleys-alleged-phone-call-americans-in-afghanistan\n",
      "1500 /transcript/hannity-on-why-biden-administration-would-give-a-dime-to-a-country-trying-to-destroy-israel\n",
      "1600 /transcript/carson-on-scrapping-obama-era-housing-rule-we-dont-need-layers-of-bureaucracy-telling-americans-how-to-live\n",
      "1700 /transcript/donald-trump-jr-calls-impeachment-investigation-a-disgrace\n",
      "1800 /transcript/graham-mueller-reached-a-conclusion-the-left-didnt-like\n",
      "1900 /transcript/troubling-cloud-of-anti-semitism-hangs-over-womens-march\n",
      "2000 /transcript/reps-jordan-meadows-react-to-strzok-page-texts-about-leaks\n",
      "2100 /transcript/hannity-president-trump-takes-bold-action-on-the-border\n",
      "2200 /transcript/doj-to-allow-fbi-informant-to-talk-about-uranium-controversy\n",
      "2300 /transcript/hannity-trump-should-replace-wh-briefings-with-tweets\n",
      "2400 /transcript/gutfeld-waukesha-parade-tragedy-jussie-smollett-trial\n",
      "2500 /transcript/gutfeld-on-rising-crime-in-democratic-cities\n",
      "2600 /transcript/tucker-biden-admin-increasing-hostility-to-crypto-is-hurting-investors\n",
      "2700 /opinion/tucker-carlson-censorship-mainstream-media\n",
      "2800 /transcript/tucker-academic-standards-in-america-are-in-freefall\n",
      "2900 /transcript/tucker-where-did-the-coronavirus-come-from\n",
      "3000 /opinion/tucker-carlson-texas-green-new-deal-climate-catastrophe\n",
      "3100 /transcript/tucker-democrats-do-nothing-to-discourage-rage-mobs\n",
      "3200 /opinion/tucker-carlson-we-must-ask-the-experts-how-they-screwed-up-the-coronavirus-models-so-badly\n",
      "3300 /media/tucker-carlson-slams-bidens-pledge-to-admit-2-million-immigrants-in-desperate-need\n",
      "3400 /opinion/tucker-carlson-ukraine-democrats-trump-impeachment-joe-biden\n",
      "3500 /opinion/tucker-carlson-nothing-about-the-resistance-to-trump-has-made-our-democracy-better-theyve-shredded-it\n",
      "3600 /opinion/tucker-carlson-what-does-beto-orourke-stand-for-who-cares-he-has-a-skateboard-and-its-awesome\n",
      "3700 /transcript/doj-charges-two-chinese-nationals-with-an-extensive-hacking-campaign\n",
      "3800 /transcript/special-report-all-star-panel-filibuster-cdc-covid-messaging\n",
      "3900 /transcript/special-report-all-star-panel-on-cdcs-updated-mask-guidance-covid-vaccine\n",
      "4000 /transcript/trump-praises-supporters-of-qanon-conspiracy-theory\n",
      "4100 /transcript/brink-of-war-democrats-react-to-soleimani-killing\n",
      "4200 /transcript/trump-administration-to-expand-expedited-deportations\n",
      "4300 /transcript/kamala-harris-releases-statement-on-jussie-smollett-charges\n",
      "4400 /transcript/politics-of-brett-kavanaughs-confirmation-fight\n",
      "4500 /transcript/will-trump-pull-the-trigger-on-military-strike-on-assad\n",
      "4600 /transcript/trump-collusion-narrative-shifts-to-clinton-and-democrats\n",
      "4700 /transcript/grapevine-princeton-university-considers-rewriting-history\n",
      "4800 /transcript/the-five-russia-ukraine-war-bidens-sotu-address\n",
      "4900 /transcript/the-five-on-terror-threats-amid-afghanistan-collapse-taliban-takeover\n",
      "5000 /opinion/gutfeld-fauci-mixed-messaging-covid\n",
      "5100 /opinion/nancy-pelosi-coronavirus-greg-gutfeld\n",
      "5200 /transcript/president-trump-declares-national-emergency-says-private-sector-stepping-up-to-combat-coronavirus\n",
      "5300 /transcript/critics-blast-abc-pundit-for-sexist-tweet-aimed-at-republican-rep-elise-stefanik\n",
      "5400 /opinion/gutfeld-on-media-coverage-of-the-mass-shootings\n",
      "5500 /opinion/gutfeld-on-cnn-whining-about-trump-skipping-the-whcd\n",
      "5600 /transcript/pelosi-urges-trump-to-delay-state-of-the-union-address-as-partial-government-shutdown-drags-on\n",
      "5700 /transcript/christine-blasey-ford-faces-her-own-allegations\n",
      "5800 /transcript/after-kim-meeting-trump-says-fake-news-is-the-biggest-enemy\n",
      "5900 /transcript/gutfeld-on-the-media-response-to-trumps-speech\n",
      "6000 /transcript/the-five-debate-the-fallout-from-flynns-deal-with-mueller\n",
      "6100 /transcript/gutfeld-on-what-the-storm-teaches-us\n",
      "6200 /transcript/kathy-griffin-plays-the-victim\n",
      "6300 /transcript/gutfeld-its-official-obama-is-scared\n",
      "6400 /transcript/gutfeld-playing-wait-and-see-with-isis-endangers-us-all\n",
      "6500 /transcript/devin-nunes-balks-at-calls-for-his-recusal-from-russia-probe\n",
      "6600 /transcript/your-world-on-infrastructure-negotiation\n",
      "6700 /transcript/dr-anthony-fauci-responds-to-congressional-critics-on-your-world\n",
      "6800 /transcript/wyoming-governor-on-canceling-states-iconic-summer-rodeo-events\n",
      "6900 /transcript/is-it-an-impeachable-offense-if-theres-no-quid-pro-quo-between-trump-and-ukraine\n",
      "7000 /transcript/new-zealand-mass-shooting-was-partially-live-streamed-on-facebook\n",
      "7100 /transcript/rep-andy-biggs-i-think-page-and-strzok-want-to-coordinate-stories\n",
      "7200 /transcript/nyse-president-farley-we-had-to-have-a-market-pullback\n",
      "7300 /transcript/bevin-lowering-the-tax-burdens-is-good-for-america\n",
      "7400 /transcript/cavuto-now-thats-a-death-spiral\n",
      "7500 /transcript/the-five-canada-truckers-vaccine-mandates\n",
      "7600 /transcript/watters-world-on-the-media-ignoring-liberal-policy-failures-and-biden-disasters-leading-to-drop-in-polls\n",
      "7700 /transcript/potential-market-fallout-over-investigations-in-washington\n",
      "7800 /opinion/greg-gutfeld-student-loan-forgiveness-squad\n",
      "7900 /opinion/greg-gutfeld-media-fulfilling-prophecy\n",
      "ERROR: 404\n",
      "8000 /transcript/rep-ratcliffe-mueller-report-proves-donald-trump-was-telling-the-truth-about-collusion\n",
      "8100 /transcript/media-buzz-on-media-afghanistan\n",
      "8200 /transcript/media-condemn-trump-putin-presser\n",
      "8300 /transcript/eric-trump-proud-of-father-fighting-every-day-for-america\n",
      "8400 /transcript/giuliani-calls-biden-a-pathological-liar-and-a-big-crook-i-cant-understand-how-hes-running-for-president\n",
      "8500 /transcript/peter-schweizer-on-media-bias-his-new-book-and-the-two-sides-to-joe-biden\n",
      "8600 /media/pennsylvania-primary-trump-biden-candidates-oz-mastriano-fetterman-shapiro-conway-hannity\n",
      "8700 /media/kudlow-biden-economy-stagflation\n",
      "8800 /media/biden-america-weak-adversaries-trump-national-security-obrien-hannity\n",
      "8900 /media/gold-star-fathers-marines-killed-kabul-biden\n",
      "9000 /media/hannity-slams-new-york-times-maggie-haberman-for-fake-pulitzer-prize\n",
      "9100 /politics/john-kennedy-public-wait-get-facts-daunte-wright-shooting\n",
      "9200 /media/sean-hannity-marjorie-taylor-greene-removed-committees\n",
      "9300 /media/hannity-biden-home-camera-hunter-email-scandal\n",
      "9400 /media/leo-terrell-democrats-black-vote-trump\n",
      "9500 /media/dhs-chad-wolf-supreme-court-daca-ruling\n",
      "9600 /media/trey-gowdy-governors-coronavirus-restrictions-rational-basis\n",
      "9700 /media/lawrence-jones-bernie-sanders-rally-michael-bloomberg\n",
      "9800 /media/sean-hannity-on-ig-report-we-were-right-every-step-of-the-way\n",
      "9900 /media/rep-gaetz-division-among-democrats-shows-that-they-have-a-losing-strategy\n",
      "10000 /politics/hogan-gidley-trump-press-secretary-parade-america-democrats\n",
      "10100 /transcript/arlen-specter-defends-party-switch-in-heated-interview\n",
      "10200 /transcript/why-did-president-obama-mislead-america-about-benghazi\n",
      "10300 /transcript/cliven-bundy-takes-on-the-nevada-ranch-rhetoric\n",
      "10400 /transcript/ditka-on-dolphins-bullying-controversy-ive-never-seen-anything-like-this-in-a-locker-room\n",
      "10500 /transcript/goolsbee-the-economy-is-heading-the-right-way\n",
      "10600 /transcript/trump-narrows-down-gop-field-as-endorsement-talk-heats-up\n",
      "10700 /transcript/could-christie-palin-paul-ryan-still-jump-in-gop-race\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10800 /transcript/nancy-pelosis-obamacare-hypocrisy\n",
      "10900 /transcript/wisconsin-governor-scott-walker-we-are-not-going-to-back-down\n",
      "11000 /transcript/rand-paul-on-the-power-of-the-tea-party\n",
      "11100 /transcript/hannity-special-investigation-obamas-most-controversial-nominee-yet\n",
      "11200 /transcript/hoekstra-leading-pack-of-challengers-in-michigan-gubernatorial-race\n",
      "11300 /transcript/exclusive-look-at-michael-steeles-new-book\n",
      "11400 /transcript/fourth-acorn-video-most-explosive-yet\n",
      "11500 /transcript/pat-robertson-on-hannity\n",
      "11600 /transcript/will-s-c-gov-sanford-turn-down-stimulus-aid\n",
      "11700 /transcript/author-debunks-numerous-popular-myths-about-global-warming\n",
      "11800 /transcript/elisabeth-hasselbeck-on-rosie-imus\n",
      "11900 /transcript/girls-gone-wild-creators-plead-guilty-to-sexual-exploitation-of-minors\n",
      "12000 /transcript/sen-john-mccain-talks-immigration-reform\n",
      "12100 /transcript/baghdad-hotel-targeted\n",
      "12200 /transcript/transcript-rev-al-sharpton\n",
      "12300 /transcript/rudy-giuliani-on-the-bush-victory\n",
      "12400 /transcript/director-and-clinton-pal-discusses-new-film\n",
      "12500 /transcript/exclusive-ed-asners-bizarre-response-to-role-in-propaganda-cartoon\n",
      "12600 /transcript/will-the-new-national-security-strategy-work\n",
      "12700 /transcript/trump-america-has-to-be-less-predictable-obama-an-unbelievable-divider\n",
      "12800 /transcript/carson-jindal-goolsbee-and-huckabee-react-to-trumps-big-indiana-win\n",
      "12900 /transcript/hannity-hillary-clinton-has-only-herself-to-blame-kellyanne-conway-on-the-latest-clinton-scandal\n",
      "13000 /media/vp-harris-refuse-invite-border-democrats-visit-mexico-issa\n",
      "13100 /media/coronavirus-elizabeth-schneider-describes-early-symptoms\n",
      "13200 /media/anti-abortion-activist-washington-dc-authorities-no-action-possible-illegal-abortions\n",
      "13300 /media/canada-conservative-rebel-media-bank-commercial-mortgage\n",
      "13400 /media/lara-logan-nicki-minaj-refusing-surrender-personal-freedoms-standing-up-dems\n",
      "13500 /media/last-days-new-york-author-liberals-de-blasio-city-downfall-crime\n",
      "13600 /media/limbaugh-guest-host-steyn-reflects-on-rushs-passing-a-brilliant-friend-whose-reach-was-unbounded\n",
      "13700 /entertainment/mark-steyn-woke-oscars-art-by-quota\n",
      "13800 /media/chicago-alderman-trump-crime-federal-agents\n",
      "13900 /media/tucker-biden-vp-choice-will-run-government\n",
      "14000 /media/rick-perry-us-oil-industry-massive-collapse\n",
      "14100 /media/tom-fitton-roger-stone-mueller-investigation\n",
      "14200 /media/charlie-hurt-steele-dossier-corrupt-news-media\n",
      "14300 /media/mark-steyn-biden-war-story-gaffe\n",
      "14400 /media/record-high-gas-prices-working-americans-livelihood-convenience-jones\n",
      "14500 /media/biden-rittenhouse-statements-watters\n",
      "14600 /media/greg-gutfeld-dana-perino-celebrate-10-year-anniversary-of-the-five\n",
      "14700 /media/judge-jeanine-pirro-trump-who-funding-coronavirus\n",
      "14800 /media/emily-compagno-all-american-citizens-lose-when-dems-refuse-to-work-with-trump\n",
      "14900 /media/alexandria-ocasio-cortez-endorses-bernie-sanders-not-game-changing-juan-williams\n",
      "15000 /shows/the-five/the-five-celebrates-8-years-memorable-moments-animals-great\n",
      "15100 /transcript/trump-takes-to-twitter-to-hit-sens-mccain-blumenthal\n",
      "15200 /transcript/president-obama-authorizes-surveillance-flights-over-syria\n",
      "15300 /transcript/victory-lap-white-house-trumpets-obamacare-enrollment\n",
      "15400 /transcript/do-media-deserve-credit-for-obamacare-disaster-bob-beckels-advice-for-president-obama\n",
      "15500 /transcript/liberal-bias-in-academia\n",
      "15600 /transcript/does-obama-have-a-plan-to-fix-the-economy\n",
      "15700 /transcript/is-fbi-faces-of-global-terrorism-ad-offensive-to-muslims\n",
      "15800 /transcript/gutfeld-obamas-pipeline-problem\n",
      "15900 /transcript/psy-gets-a-pass\n",
      "16000 /transcript/fallout-from-benjamin-netanyahus-election-victory\n",
      "16100 /transcript/ben-carson-outlines-immigration-plan\n",
      "16200 /transcript/republican-candidates-set-to-square-off-at-fox-news-debate\n",
      "16300 /transcript/tata-presidential-briefer-told-centcom-to-alter-isis-intel\n",
      "16400 /media/sen-rubio-likeliest-scenario-russias-chemical-biological-weapons\n",
      "16500 /politics/mccarthy-rips-pelosi-trump-coronavirus-diagnosis\n",
      "16600 /transcript/does-the-president-have-an-obama-doctrine\n",
      "16700 /transcript/libertarian-nominee-gary-johnson-makes-his-pitch-to-voters\n",
      "16800 /media/jesse-watters-trump-has-won-every-democrat-debate\n",
      "16900 /media/cuomo-bill-clinton-playbook-marc-thiessen\n",
      "17000 /media/russia-ukraine-war-glenn-greenwald-biden-putin\n",
      "17100 /media/ingraham-lessons-canada-freedom-convoy\n",
      "17200 /media/josh-hawley-ingraham-democrats-woke-politics-hurting-working-families\n",
      "17300 /media/laura-ingraham-democrat-policies-crime-surge\n",
      "17400 /media/ingraham-biden-fauci-control-freaks-america-under-thumb\n",
      "17500 /media/ingraham-vice-presidential-debate-reaction\n",
      "17600 /media/dr-charles-lockwood-florida-coronavirus-cases-increase\n",
      "17700 /media/mollie-hemingway-obama-comey-trump-campaign-spying\n",
      "17800 /media/roger-stone-juror-controversy-matt-whitaker-responds\n",
      "17900 /media/jeff-sessions-makes-his-case-for-trumps-support-in-alabama-senate-race-warns-democrats-of-the-danger-of-impeachment-inquiry\n",
      "18000 /travel/texas-man-intervened-violent-disneyland-brawl\n",
      "18100 /media/maga-rally-violence-antifa-coverage-jason-miller\n",
      "18200 /transcript/donald-trumps-massive-media-blitz-media-mark-sanders-surge\n",
      "18300 /media/herschel-walker-says-not-playing-college-football-a-political-decision-athletes-safer-on-campus\n",
      "18400 /media/milley-call-china-aggressive-gordon-chang\n",
      "18500 /media/civil-rights-leader-slams-diabolical-new-york-times-1619-project-white-supremacy-to-assume-all-african-americans-are-victims\n",
      "18600 /politics/bret-baier-biden-harris-dems-at-ease\n",
      "18700 /media/ohio-gov-dewine-we-cant-gather-together\n",
      "18800 /transcript/national-counterintelligence-exec-on-going-after-leakers\n",
      "18900 /transcript/cavuto-it-helps-to-be-smart-but-doesnt-hurt-to-smile\n",
      "19000 /transcript/cavuto-obamacare-food-for-thought\n",
      "19100 /transcript/cavuto-bad-stuff-on-romney-matters-to-mainstream-media\n",
      "19200 /transcript/cavuto-are-we-still-independent\n",
      "19300 /transcript/health-cares-fine-print-fallout\n",
      "19400 /transcript/whats-in-a-political-name\n",
      "19500 /transcript/sometimes-the-cure-can-be-worse-than-the-disease\n",
      "19600 /transcript/if-nothing-else-donald-trump-is-clear\n",
      "19700 /transcript/getting-something-out-of-nothing\n",
      "19800 /transcript/the-magic-of-compounding-and-dollar-cost-averaging\n",
      "19900 /transcript/big-oils-slippery-slope\n",
      "20000 /transcript/do-the-right-thing\n",
      "20100 /transcript/not-so-stupid-americans\n",
      "20200 /transcript/going-down-swinging\n",
      "20300 /transcript/american-first-journalist-second\n",
      "20400 /transcript/isnt-that-rich\n",
      "20500 /transcript/amb-bolton-critiques-nikki-haleys-confirmation-hearing\n",
      "20600 /transcript/owner-of-looted-baltimore-shoe-store-on-handling-of-crisis\n",
      "20700 /transcript/mcconnell-clinton-democrat-same-as-obama-democrat\n",
      "20800 /transcript/louisiana-senate-candidate-wrestles-gator-in-tv-ad\n",
      "20900 /transcript/states-divided-over-letting-insurers-extend-canceled-plans\n",
      "21000 /transcript/bachmann-tea-party-more-energized-than-ever-before\n",
      "21100 /transcript/ceos-call-for-reforms-to-entitlements\n",
      "21200 /transcript/cavuto-lets-start-demanding-more-from-our-government\n",
      "21300 /transcript/rubio-every-year-we-wait-the-harder-it-is-to-solve\n",
      "21400 /transcript/report-companies-added-325000-jobs-in-december\n",
      "21500 /transcript/fmr-verizon-wireless-ceo-cutting-corporate-taxes-will-create-jobs\n",
      "21600 /transcript/huckabee-people-just-want-work\n",
      "21700 /transcript/south-carolina-congressional-candidate-tim-scott-defends-cutting-spending\n",
      "21800 /transcript/democrats-say-ditch-bush-tax-cuts\n",
      "21900 /transcript/rep-kucinich-on-ted-kennedys-passing-and-health-care\n",
      "22000 /transcript/fred-thompson-on-obamas-1-trillion-bank-plan-growing-calls-for-geithners-resignation\n",
      "22100 /transcript/democratic-north-carolina-congressman-brad-miller-on-700b-bailout-plan\n",
      "22200 /transcript/sen-lieberman-criticizes-obama\n",
      "22300 /transcript/former-federal-reserve-chairman-alan-greenspan-on-interest-rates-bush-economic-policy-personal-legacy\n",
      "22400 /transcript/peggy-noonan-on-political-impact-of-illegal-immigration\n",
      "22500 /transcript/transcript-start-bragging\n",
      "22600 /transcript/transcript-toll-brothers-bob-toll\n",
      "22700 /transcript/entertaining-the-troops\n",
      "22800 /transcript/fox-news-exclusive-banking-behemoth\n",
      "22900 /transcript/treasury-secretary-john-snow-on-iraq-economy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23000 /transcript/colin-foster-pres-of-bayer-north-america-andchris-viehbacher-pres-of-glaxosmithkline-u-s\n",
      "23100 /transcript/larry-namer-co-founder-of-reality-central\n",
      "23200 /transcript/jeb-bush-rates-his-debate-performance-i-thought-i-did-great\n",
      "23300 /transcript/jindal-on-trump-this-man-is-not-a-real-conservative\n",
      "23400 /transcript/gov-greg-abbott-team-america-must-come-first\n",
      "23500 /transcript/do-you-wake-up-in-a-sweat-overcome-with-regret\n",
      "23600 /transcript/sen-mike-lee-says-he-couldnt-agree-more-with-president-trumps-decision-to-withdraw-all-us-troops-from-syria\n",
      "23700 /politics/biden-coronavirus-recovery-trillion-democrats-infrastructure\n",
      "23800 /transcript/mike-pence-on-travel-restrictions-supreme-court-pick-sen-feinstein-on-challenges-to-trumps-executive-actions\n",
      "23900 /transcript/rep-chris-van-hollen-rep-jim-jordan-on-the-fiscal-fight-still-ahead\n",
      "24000 /transcript/reps-clyburn-mcmorris-rodgers-sens-paul-coons-react-to-arizona-rampage\n",
      "24100 /transcript/transcript-gen-petraeus-on-fns\n",
      "24200 /transcript/transcript-sen-dick-durbin-on-fox-news-sunday\n",
      "24300 /transcript/transcript-justice-stephen-breyer-sits-down-with-fox-news-sunday\n",
      "24400 /transcript/transcript-gov-bill-richardson-on-fns\n",
      "24500 /transcript/gingrich-on-trumps-attacks-on-trump-university-judge-john-podesta-on-clintons-strategies-against-sanders-trump\n"
     ]
    }
   ],
   "source": [
    "for i, r in df.iterrows():\n",
    "    fn = 'html/' + r['html_file']\n",
    "    if os.path.exists(fn):\n",
    "        continue\n",
    "    url = r['url']\n",
    "    headers = {\"User-Agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36\"}\n",
    "    try:\n",
    "        rq = requests.get('https://www.foxnews.com/' + url, headers=headers)\n",
    "        if rq.status_code == 200:\n",
    "            with open(fn, 'wb') as f:\n",
    "                f.write(rq.content)\n",
    "        else:\n",
    "            print('ERROR:', rq.status_code)\n",
    "    except Exception as e:\n",
    "        print('ERROR:', e)\n",
    "    time.sleep(0.5)\n",
    "    if i % 100 == 0:\n",
    "        print(i, url)\n",
    "    #break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "ed8998fb",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('foxnews-transcript.csv.gz', compression='gzip', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b00cc325",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3ab18e3c",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}