02.2 Extract Raw Data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import requests\n",
    "import pattern3.web as web\n",
    "import pandas as pd \n",
    "import numpy as np \n",
    "import matplotlib.pyplot as plt\n",
    "from progressbar import ProgressBar\n",
    "from bs4 import BeautifulSoup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import errno"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "<br><Br/>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def get_episode_description(webpage, i):\n",
    "    \n",
    "    \n",
    "    try: \n",
    "        url = web.URL(webpage)\n",
    "        bs = BeautifulSoup(url.download(cached = False), features='xml') \n",
    "\n",
    "        podcast_titles = [] \n",
    "        descriptions_processed = []\n",
    "\n",
    "        titles = bs.findAll('title')\n",
    "        descriptions = bs.findAll('description')\n",
    "\n",
    "\n",
    "        if (titles is not None) and (titles != []): \n",
    "            for title in titles:\n",
    "                title = title.getText()\n",
    "                if \"<\" in title:  \n",
    "                    bs2 = BeautifulSoup(title)\n",
    "                    title = bs2.getText() \n",
    "                    if title not in podcast_titles:\n",
    "                        with open(str(i) + '.txt','a') as t:\n",
    "                            t.write('\\n' + title)      \n",
    "                            t.close()\n",
    "\n",
    "                else:    \n",
    "                    if title not in podcast_titles:\n",
    "                        with open(str(i) +'.txt','a') as t:\n",
    "                            t.write('\\n' + title)      \n",
    "                            t.close()\n",
    "        else:\n",
    "            with open(str(i) + '.txt','a') as t:\n",
    "                t.write('\\n' + '')      \n",
    "                t.close() \n",
    "            \n",
    "            \n",
    "        if (descriptions is not None) and (descriptions != []):\n",
    "            for description in descriptions:\n",
    "                description = description.getText()\n",
    "\n",
    "\n",
    "                if \"<\" in description:  \n",
    "                    bs2 = BeautifulSoup(description)\n",
    "                    description = bs2.getText()  \n",
    "                    with open(str(i) + '.txt','a') as f:\n",
    "                        f.write('\\n' + description)      \n",
    "                        f.close()\n",
    "\n",
    "                else: \n",
    "                    with open(str(i) +'.txt','a') as f:\n",
    "                        f.write('\\n' + description)      \n",
    "                        f.close()\n",
    "        else:\n",
    "            with open(str(i) + '.txt','a') as t:\n",
    "                t.write('\\n' + '')      \n",
    "                t.close() \n",
    "                \n",
    "                \n",
    "                \n",
    "        \n",
    "    except: \n",
    "        with open(str(i) + '.txt','a') as t:\n",
    "            t.write('\\n' + 'empty')      \n",
    "            t.close() \n",
    "\n",
    "    return"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dfpod = pd.read_csv('df_popular_podcasts.csv', index_col = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/odeniz/Downloads/ENTER/lib/python3.6/site-packages/bs4/__init__.py:181: UserWarning: No parser was explicitly specified, so I'm using the best available HTML parser for this system (\"lxml\"). This usually isn't a problem, but if you run this code on another system, or in a different virtual environment, it may use a different parser and behave differently.\n",
      "\n",
      "The code that caused this warning is on line 193 of the file /Users/odeniz/Downloads/ENTER/lib/python3.6/runpy.py. To get rid of this warning, change code that looks like this:\n",
      "\n",
      " BeautifulSoup([your markup])\n",
      "\n",
      "to this:\n",
      "\n",
      " BeautifulSoup([your markup], \"lxml\")\n",
      "\n",
      "  markup_type=markup_type))\n"
     ]
    }
   ],
   "source": [
    "pbar = ProgressBar()\n",
    "for i, link in enumerate(dfpod['Feed URL']):\n",
    "    get_episode_description(link, i)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}