get_tweets.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Getting tweets by philosophers\n",
    "\n",
    "This notebooks shows how you can build a dataset of philosopher-tweets from the list given by @truesciphi.\n",
    "It gets the accounts out of the html of the list, then downloads their timelines using twarc (Which you will need to configure:  https://github.com/DocNow/twarc) and extracts who the accounts have been mentioning."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import seaborn as sns; sns.set()\n",
    "import matplotlib.pyplot as plt\n",
    "from bs4 import BeautifulSoup\n",
    "import requests\n",
    "import re\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "Load the philo-twitter-list:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# url: truesciphi.org/phi.html\n",
    "soup = BeautifulSoup(open(\"philotwitter_2.html\",encoding='utf-8'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This code did oversample. I corrected this in the next notebook. If you run this, you should use the correct one:\n",
    "# links = []\n",
    "# for link in soup.findAll('a'):\n",
    "#     links.append(link.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This is the correct code:\n",
    "soup = BeautifulSoup(open(\"philotwitter_2.html\",encoding='utf-8'))\n",
    "links = []\n",
    "for link in soup.findAll('a'):\n",
    "    links.append(link.text)\n",
    "\n",
    "r1 = re.findall(r\"@[A-Za-z0-9_]*\",str(links))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "r1 = re.findall(r\"@[A-Za-z0-9_]*(?=<)\",str(soup))\n",
    "len(r1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We dump our list, in case we need it later:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "pickle.dump( r1, open( \"philosophen_liste.p\", \"wb\" ) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "philos = pickle.load( open( \"philosophen_liste.p\", \"rb\" ) )\n",
    "print(philos)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we send a command to twarc (in the console) that downloads from the timeline of the user. I don't know exactly what determines how much it gets, but it's usually around 3000 tweets. It's roughly ten GB, so I pointed this to an external drive."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "count = 0\n",
    "\n",
    "for tweeter in philos:\n",
    "    filename = str(r'E:/timelines/' + tweeter + '.jsonl')\n",
    "    name = tweeter.replace('@','')\n",
    "    print(count)\n",
    "    count = count +1\n",
    "    !twarc timeline $name > $filename\n",
    "#     !twarc users $name > $filename"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We loop over all the collected files, and extract for each author the metadata associated with the last tweet. We also extract lists with all the mentioned accounts (from the 'user_mentions' column). You could do way more with this, e. g. by extracting texts and building your similarities from that."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from IPython.display import display\n",
    "import os\n",
    "from io import StringIO\n",
    "import tqdm\n",
    "\n",
    "tweetcounts = []\n",
    "tweeter_data = []\n",
    "\n",
    "for filename in tqdm.tqdm_notebook(os.listdir(r'E:/timelines/')):\n",
    "    print(str(r'E:/timelines/'+filename))\n",
    "    with open(str(r'E:/timelines/'+filename), 'r') as file:\n",
    "        try:\n",
    "            data = file.read()\n",
    "            this_dataset = pd.read_json(StringIO(data), lines=True)\n",
    "            tweetcounts.append(len(this_dataset))\n",
    "            mentioned_users_screenames = [[y['screen_name'] for y in x['user_mentions']] for x in this_dataset['entities']]\n",
    "            mentioned_users_screenames = [item for sublist in mentioned_users_screenames for item in sublist]\n",
    "\n",
    "            mentioned_users_names = [[y['name'] for y in x['user_mentions']] for x in this_dataset['entities']]\n",
    "            mentioned_users_names = [item for sublist in mentioned_users_names for item in sublist]\n",
    "\n",
    "            user_values = this_dataset['user'][0]\n",
    "            user_values.pop('entities') # we need to remove these, because they mess uo the df otherwise.\n",
    "\n",
    "\n",
    "            user_data = pd.DataFrame(user_values, index=[0])\n",
    "            user_data['mentioned_users_screenames'] = str(mentioned_users_screenames)\n",
    "            user_data['mentioned_users_names'] = str(mentioned_users_names)\n",
    "            tweeter_data.append(user_data)\n",
    "        except:\n",
    "            print('Parsing failure') # These are mainly protected accounts\n",
    "            pass\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# We have a look at how meny tweets there are.\n",
    "print(np.sum(tweetcounts))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# And now we built the whole dataset, have a look at it, and pickle it.\n",
    "tweeter_data = pd.concat(tweeter_data)\n",
    "display(tweeter_data)\n",
    "pickle.dump( tweeter_data, open( \"tweeter_data.p\", \"wb\" ) )"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}