reuters.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import sys\n",
    "import time\n",
    "import datetime\n",
    "import util\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "\n",
    "class ReutersCrawler(object):\n",
    "\n",
    "    def __init__(self):\n",
    "        self.ticker_list_filename = './input/apple_ticker.csv'\n",
    "        self.news_filename = './input/news_reuters.csv'\n",
    "\n",
    "    def fetch_content(self, task, date_range):\n",
    "        # https://uk.reuters.com/info/disclaimer\n",
    "        ticker, name, exchange, market_cap = task\n",
    "        print(\"%s - %s - %s - %s\" % (ticker, name, exchange, market_cap))\n",
    "\n",
    "        suffix = {'AMEX': '.A', 'NASDAQ': '.O', 'NYSE': '.N'}\n",
    "        # e.g. https://www.reuters.com/finance/stocks/company-news/BIDU.O?date=09262017\n",
    "        \n",
    "#         url = \"https://www.reuters.com/finance/stocks/company-news/AAPL.O\"\n",
    "        url = \"https://www.reuters.com/finance/stocks/company-news/\" + ticker.strip() + suffix[exchange]\n",
    "        print(url)\n",
    "        today = datetime.datetime.today().strftime(\"%Y%m%d\")\n",
    "\n",
    "        news_num = self.get_news_num_whenever(url)\n",
    "        print(news_num)\n",
    "        if news_num:\n",
    "            # this company has news, then fetch for N consecutive days in the past\n",
    "            has_content, no_news_days = self.fetch_within_date_range(news_num, url, date_range, task, ticker)\n",
    "            if not has_content:\n",
    "                print('%s has no content within date range' % ticker)\n",
    "        else:\n",
    "            print(\"%s has no news at all, set as LOWEST priority\" % (ticker))\n",
    "    \n",
    "    def get_news_num_whenever(self, url):\n",
    "        # check the website to see if the ticker has any news\n",
    "        # return the number of news\n",
    "        soup = util.get_soup_with_repeat(url, repeat_times=4)\n",
    "        if soup:\n",
    "            return len(soup.find_all(\"div\", {'class': ['topStory', 'feature']}))\n",
    "        return 0\n",
    "\n",
    "    def fetch_within_date_range(self, news_num, url, date_range, task, ticker):\n",
    "        # if it doesn't have a single news for X consecutive days, stop iterating dates\n",
    "        # set this ticker into the second-lowest priority list\n",
    "        missing_days = 0\n",
    "        has_content = False\n",
    "        no_news_days = []\n",
    "        for timestamp in date_range:\n",
    "            print('trying '+timestamp)  # print timestamp on the same line\n",
    "            new_time = timestamp[4:] + timestamp[:4] # change 20151231 to 12312015 to match reuters format\n",
    "            soup = util.get_soup_with_repeat(url + \"?date=\" + new_time)\n",
    "            if soup and self.parse_and_save_news(soup, task, ticker, timestamp):\n",
    "                missing_days = 0 # if get news, reset missing_days as 0\n",
    "                has_content = True\n",
    "            else:\n",
    "                missing_days += 1\n",
    "\n",
    "        return has_content, no_news_days\n",
    "\n",
    "    def parse_and_save_news(self, soup, task, ticker, timestamp):\n",
    "        content = soup.find_all(\"div\", {'class': ['topStory', 'feature']})\n",
    "        if not content:\n",
    "            return False\n",
    "        with open(self.news_filename, 'a+', newline='\\n') as fout:\n",
    "            for i in range(len(content)):\n",
    "                title = content[i].h2.get_text().replace(\",\", \" \").replace(\"\\n\", \" \")\n",
    "                body = content[i].p.get_text().replace(\",\", \" \").replace(\"\\n\", \" \")\n",
    "\n",
    "                if i == 0 and soup.find_all(\"div\", class_=\"topStory\"):\n",
    "                    news_type = 'topStory'\n",
    "                else:\n",
    "                    news_type = 'normal'\n",
    "\n",
    "                try:\n",
    "                    fout.write(','.join([ticker, task[1], timestamp, title, body, news_type]) + '\\n')\n",
    "                except:\n",
    "                    pass\n",
    "#                 fout.write(','.join([ticker, task[1], timestamp, title, body, news_type])+ '\\n')\n",
    "        return True\n",
    "\n",
    "    def run(self, numdays=1000, start_date='12/31/1999'):\n",
    "        \"\"\"Start crawler back to numdays\"\"\"\n",
    "        date_range = util.generate_past_n_days(numdays, start_date) # look back on the past X days\n",
    "#         print(date_range)\n",
    "        # store low-priority task and run later\n",
    "        delayed_tasks = {'LOWEST': set(), 'LOW': set()}\n",
    "        with open(self.ticker_list_filename, encoding=\"utf-8\") as ticker_list:\n",
    "            for line in ticker_list:  # iterate all possible tickers\n",
    "                task = tuple(line.strip().split(','))\n",
    "                if len(task) < 4:\n",
    "                    continue\n",
    "                ticker, name, exchange, market_cap = task\n",
    "                if ticker in finished_tickers:\n",
    "                    continue\n",
    "                if ticker in failed_tickers:\n",
    "                    priority = failed_tickers[ticker]\n",
    "                    delayed_tasks[priority].add(task)\n",
    "                    continue\n",
    "                self.fetch_content(task, date_range)\n",
    "\n",
    "        # run task with low priority\n",
    "        for task in delayed_tasks['LOW']:\n",
    "            self.fetch_content(task, date_range)\n",
    "        # run task with lowest priority\n",
    "        for task in delayed_tasks['LOWEST']:\n",
    "            self.fetch_content(task, date_range)\n",
    "\n",
    "\n",
    "def main():\n",
    "    reuter_crawler = ReutersCrawler()\n",
    "    reuter_crawler.run(6847, '07/12/2017')\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    main()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}