Data_Download_Script.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/site-packages/ipykernel_launcher.py:77: DeprecationWarning: insert is deprecated. Use insert_one or insert_many instead.\n",
      "/usr/local/lib/python3.6/site-packages/ipykernel_launcher.py:141: DeprecationWarning: insert is deprecated. Use insert_one or insert_many instead.\n"
     ]
    }
   ],
   "source": [
    "import tweepy,pymongo\n",
    "import json\n",
    "import pandas as pd\n",
    "\n",
    "conskeys = ######Insert key\n",
    "consecret = #####Insert secret\n",
    "acctok = #####Insert token\n",
    "acctoksecret = #####Insert secret token\n",
    "\n",
    "# Authentication\n",
    "auth = tweepy.OAuthHandler(conskeys, consecret)\n",
    "auth.set_access_token(acctok, acctoksecret)\n",
    "api = tweepy.API(auth)\n",
    "query = \"U.S. economy\"\n",
    "\n",
    "client = pymongo.MongoClient()\n",
    "db = client[\"tweet_database\"]\n",
    "my_tweets_table = db.tweets_collection_final  # Setting collection to enter documents.\n",
    "\n",
    "_max_queries = 175  # arbitrarily chosen value\n",
    "tweet = tweet_batch = api.search(q=query, count=100,tweet_mode='extended')\n",
    "\n",
    "for tweet in tweet_batch:\n",
    "        tweets = []\n",
    "        tweets.append((tweet.created_at,tweet.id,tweet.full_text,tweet.source, tweet.truncated, tweet.metadata,tweet.in_reply_to_status_id, tweet.in_reply_to_user_id, tweet.in_reply_to_screen_name,tweet.user._json, tweet.place,tweet.is_quote_status, tweet.retweet_count,tweet.favorite_count,tweet.entities,tweet.favorited,tweet.retweeted,tweet.lang))\n",
    "        tweetsdf = pd.DataFrame(tweets,columns=['created_at', 'id', 'full_text', 'source', 'truncated','metadata','in_reply_to_status_id','in_reply_to_user_id','in_reply_to_screen_name', 'user','place','is_quote_status','retweet_count','favorite_count', 'entities','favorited','retweeted','lang'])\n",
    "        tweetsdf.to_csv('Economy_USA_final.csv')\n",
    "        id_ = tweet.id\n",
    "        id_str = tweet.id_str\n",
    "        user_id = tweet.user.id_str\n",
    "        user_name = tweet.user.name\n",
    "        user = tweet.user._json\n",
    "        in_reply_to_user_id = tweet.in_reply_to_user_id\n",
    "        lang = tweet.lang\n",
    "        source = tweet.source\n",
    "        retweet_count = tweet.retweet_count\n",
    "        in_reply_to_status_id = tweet.in_reply_to_status_id\n",
    "        lang = tweet.lang\n",
    "        fav = tweet.favorite_count\n",
    "        try:\n",
    "            x = tweet.retweeted_status  \n",
    "            retweetedFrom_id = x.user.id_str\n",
    "            retweetedFrom_OrigTweetid = x.id\n",
    "            retweetedFrom_name = x.user.screen_name\n",
    "            RT = True\n",
    "        except AttributeError:  \n",
    "            retweetedFrom_id = -1\n",
    "            retweetedFrom_name = 'NA'\n",
    "            retweetedFrom_OrigTweetid = -1\n",
    "            RT = False\n",
    "        if RT:\n",
    "            content = tweet.retweeted_status.full_text\n",
    "            retweet_fav = tweet.retweeted_status.favorite_count\n",
    "        else:\n",
    "            content = tweet.full_text\n",
    "            retweet_fav = -1\n",
    "        created_at = str(tweet.created_at)\n",
    "        location = tweet.user.location\n",
    "        ht = tweet.entities[\"hashtags\"]\n",
    "        hashtags = []\n",
    "        for i in ht:\n",
    "            hashtags.append(i[\"text\"]) # List of hashtags.\n",
    "        mntn = tweet.entities[\"user_mentions\"]\n",
    "        mentions = []\n",
    "        for i in mntn:\n",
    "            mentions.append(i['id_str']) # List of mentions\n",
    "        try:\n",
    "            md = tweet.entities[\"media\"]\n",
    "            media = []\n",
    "            for i in md:\n",
    "                media.append(i[\"url\"]) # And list of media (photos only)\n",
    "        except KeyError:  # If no media found\n",
    "            media = []\n",
    "        try:\n",
    "            jsonx = {\"id\":id_,\"user_name\": user_name, \"user_id\": user_id, \"content\": content, \"created_at\": created_at, \"location\": location,\n",
    "            \"hashtags\": hashtags, \"mentions\": mentions, \"in_reply_to_user_id\": in_reply_to_user_id,\"in_reply_to_status_id\":in_reply_to_status_id,\"retweedt_orid_tweetID\": retweetedFrom_OrigTweetid,\"retweetedFrom_id\": retweetedFrom_id,\"retweetedFrom_name\": retweetedFrom_name, \"media\": media, \"retweet\": RT, \"FavCount\": fav,\"Orig_retweet_fav\":retweet_fav ,\"source\":source, \"retweet_count\":retweet_count, \"lang\":lang} # Making a JSON to be sent to collection.\n",
    "            my_tweets_table.insert(jsonx)  # And insert\n",
    "        except pymongo.errors.DuplicateKeyError as e:  # Except if it's already there\n",
    "            print(\"Duplicate Key\")\n",
    "\n",
    "ct = 1\n",
    "while ct < _max_queries:\n",
    "    tweet_batch = api.search(q=query, \n",
    "                             count=100,\n",
    "                             max_id=tweet_batch.max_id-1,tweet_mode='extended')\n",
    "    \n",
    "    for tweet in tweet_batch:\n",
    "        tweets = []\n",
    "        tweets.append((tweet.created_at,tweet.id,tweet.full_text,tweet.source, tweet.truncated, tweet.metadata, tweet.in_reply_to_status_id, tweet.in_reply_to_user_id, tweet.in_reply_to_screen_name, tweet.user._json, tweet.place,tweet.is_quote_status, tweet.retweet_count,tweet.favorite_count,tweet.entities,tweet.favorited,tweet.retweeted, tweet.lang ))\n",
    "        tweetsdf = pd.DataFrame(tweets,columns=['created_at', 'id', 'full_text', 'source', 'truncated','metadata','in_reply_to_status_id','in_reply_to_user_id','in_reply_to_screen_name','user','place','is_quote_status','retweet_count','favorite_count','entities','favorited','retweeted','lang'])\n",
    "        tweetsdf.to_csv('Economy_USA_final.csv', mode='a', header=False)\n",
    "        id_ = tweet.id\n",
    "        id_str = tweet.id_str\n",
    "        user_id = tweet.user.id_str\n",
    "        user_name = tweet.user.name\n",
    "        user = tweet.user._json\n",
    "        in_reply_to_user_id = tweet.in_reply_to_user_id\n",
    "        lang = tweet.lang\n",
    "        source = tweet.source\n",
    "        retweet_count = tweet.retweet_count\n",
    "        in_reply_to_status_id = tweet.in_reply_to_status_id\n",
    "        lang = tweet.lang\n",
    "        fav = tweet.favorite_count\n",
    "        try:\n",
    "            x = tweet.retweeted_status  \n",
    "            retweetedFrom_id = x.user.id_str\n",
    "            retweetedFrom_name = x.user.screen_name\n",
    "            retweetedFrom_OrigTweetid = x.id\n",
    "            RT = True\n",
    "        except AttributeError:  \n",
    "            retweetedFrom_id = -1\n",
    "            retweetedFrom_name = 'NA'\n",
    "            retweetedFrom_OrigTweetid = -1\n",
    "            RT = False\n",
    "        if RT:\n",
    "            content = tweet.retweeted_status.full_text\n",
    "            retweet_fav = tweet.retweeted_status.favorite_count\n",
    "        else:\n",
    "            content = tweet.full_text\n",
    "            retweet_fav = -1\n",
    "        created_at = str(tweet.created_at)\n",
    "        location = tweet.user.location\n",
    "        ht = tweet.entities[\"hashtags\"]\n",
    "        hashtags = []\n",
    "        for i in ht:\n",
    "            hashtags.append(i[\"text\"]) # List of hashtags.\n",
    "        mntn = tweet.entities[\"user_mentions\"]\n",
    "        mentions = []\n",
    "        for i in mntn:\n",
    "            mentions.append(i['id_str']) # List of mentions\n",
    "        try:\n",
    "            md = tweet.entities[\"media\"]\n",
    "            media = []\n",
    "            for i in md:\n",
    "                media.append(i[\"url\"]) # And list of media (photos only)\n",
    "        except KeyError:  # If no media found\n",
    "            media = []\n",
    "        try:\n",
    "            jsonx = {\"id\":id_,\"user_name\": user_name, \"user_id\": user_id, \"content\": content, \"created_at\": created_at, \"location\": location,\n",
    "            \"hashtags\": hashtags, \"mentions\": mentions, \"in_reply_to_user_id\": in_reply_to_user_id,\"in_reply_to_status_id\":in_reply_to_status_id,\"retweedt_orid_tweetID\": retweetedFrom_OrigTweetid,\"retweetedFrom_id\": retweetedFrom_id,\"retweetedFrom_name\": retweetedFrom_name, \"media\": media, \"retweet\": RT, \"FavCount\": fav,\"Orig_retweet_fav\":retweet_fav ,\"source\":source, \"retweet_count\":retweet_count, \"lang\":lang} # Making a JSON to be sent to collection.\n",
    "            my_tweets_table.insert(jsonx)  # And insert\n",
    "        except pymongo.errors.DuplicateKeyError as e:  # Except if it's already there\n",
    "            print(\"Duplicate Key\")\n",
    "    ct += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>created_at</th>\n",
       "      <th>id</th>\n",
       "      <th>full_text</th>\n",
       "      <th>source</th>\n",
       "      <th>truncated</th>\n",
       "      <th>metadata</th>\n",
       "      <th>in_reply_to_status_id</th>\n",
       "      <th>in_reply_to_user_id</th>\n",
       "      <th>in_reply_to_screen_name</th>\n",
       "      <th>user</th>\n",
       "      <th>place</th>\n",
       "      <th>is_quote_status</th>\n",
       "      <th>retweet_count</th>\n",
       "      <th>favorite_count</th>\n",
       "      <th>entities</th>\n",
       "      <th>favorited</th>\n",
       "      <th>retweeted</th>\n",
       "      <th>lang</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>2020-04-29 18:05:38</td>\n",
       "      <td>1255558878403547136</td>\n",
       "      <td>FED SAYS IT IS COMMITTED TO USING ITS FULL RAN...</td>\n",
       "      <td>Twitter Web App</td>\n",
       "      <td>False</td>\n",
       "      <td>{'iso_language_code': 'en', 'result_type': 're...</td>\n",
       "      <td>1.255559e+18</td>\n",
       "      <td>1.040666e+18</td>\n",
       "      <td>RedboxWire</td>\n",
       "      <td>{'id': 1040665819187953664, 'id_str': '1040665...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>{'hashtags': [], 'symbols': [], 'user_mentions...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>en</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>2020-04-29 18:05:33</td>\n",
       "      <td>1255558857683804160</td>\n",
       "      <td>RT @CGTNOfficial: The Federal Reserve painted ...</td>\n",
       "      <td>Twitter for iPhone</td>\n",
       "      <td>False</td>\n",
       "      <td>{'iso_language_code': 'en', 'result_type': 're...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{'id': 498137972, 'id_str': '498137972', 'name...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>{'hashtags': [], 'symbols': [], 'user_mentions...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>en</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>2020-04-29 18:05:16</td>\n",
       "      <td>1255558785613074435</td>\n",
       "      <td>THREAD—\"The Federal Reserve is committed to us...</td>\n",
       "      <td>Twitter Web App</td>\n",
       "      <td>False</td>\n",
       "      <td>{'iso_language_code': 'en', 'result_type': 're...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{'id': 65466158, 'id_str': '65466158', 'name':...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>{'hashtags': [], 'symbols': [], 'user_mentions...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>en</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>2020-04-29 18:05:12</td>\n",
       "      <td>1255558770291335168</td>\n",
       "      <td>The U.S. Government Out Here Carring About It’...</td>\n",
       "      <td>Twitter for iPhone</td>\n",
       "      <td>False</td>\n",
       "      <td>{'iso_language_code': 'en', 'result_type': 're...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{'id': 1399854920, 'id_str': '1399854920', 'na...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>{'hashtags': [], 'symbols': [], 'user_mentions...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>en</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>2020-04-29 18:05:09</td>\n",
       "      <td>1255558756890374144</td>\n",
       "      <td>“The Federal Reserve said the U.S. economy has...</td>\n",
       "      <td>Twitter for iPhone</td>\n",
       "      <td>False</td>\n",
       "      <td>{'iso_language_code': 'en', 'result_type': 're...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{'id': 940032448057896960, 'id_str': '94003244...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>{'hashtags': [], 'symbols': [], 'user_mentions...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>en</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0           created_at                   id  \\\n",
       "0           0  2020-04-29 18:05:38  1255558878403547136   \n",
       "1           0  2020-04-29 18:05:33  1255558857683804160   \n",
       "2           0  2020-04-29 18:05:16  1255558785613074435   \n",
       "3           0  2020-04-29 18:05:12  1255558770291335168   \n",
       "4           0  2020-04-29 18:05:09  1255558756890374144   \n",
       "\n",
       "                                           full_text              source  \\\n",
       "0  FED SAYS IT IS COMMITTED TO USING ITS FULL RAN...     Twitter Web App   \n",
       "1  RT @CGTNOfficial: The Federal Reserve painted ...  Twitter for iPhone   \n",
       "2  THREAD—\"The Federal Reserve is committed to us...     Twitter Web App   \n",
       "3  The U.S. Government Out Here Carring About It’...  Twitter for iPhone   \n",
       "4  “The Federal Reserve said the U.S. economy has...  Twitter for iPhone   \n",
       "\n",
       "   truncated                                           metadata  \\\n",
       "0      False  {'iso_language_code': 'en', 'result_type': 're...   \n",
       "1      False  {'iso_language_code': 'en', 'result_type': 're...   \n",
       "2      False  {'iso_language_code': 'en', 'result_type': 're...   \n",
       "3      False  {'iso_language_code': 'en', 'result_type': 're...   \n",
       "4      False  {'iso_language_code': 'en', 'result_type': 're...   \n",
       "\n",
       "   in_reply_to_status_id  in_reply_to_user_id in_reply_to_screen_name  \\\n",
       "0           1.255559e+18         1.040666e+18              RedboxWire   \n",
       "1                    NaN                  NaN                     NaN   \n",
       "2                    NaN                  NaN                     NaN   \n",
       "3                    NaN                  NaN                     NaN   \n",
       "4                    NaN                  NaN                     NaN   \n",
       "\n",
       "                                                user place  is_quote_status  \\\n",
       "0  {'id': 1040665819187953664, 'id_str': '1040665...   NaN            False   \n",
       "1  {'id': 498137972, 'id_str': '498137972', 'name...   NaN            False   \n",
       "2  {'id': 65466158, 'id_str': '65466158', 'name':...   NaN            False   \n",
       "3  {'id': 1399854920, 'id_str': '1399854920', 'na...   NaN            False   \n",
       "4  {'id': 940032448057896960, 'id_str': '94003244...   NaN            False   \n",
       "\n",
       "   retweet_count  favorite_count  \\\n",
       "0              1               4   \n",
       "1              4               0   \n",
       "2              3               3   \n",
       "3              0               0   \n",
       "4              0               1   \n",
       "\n",
       "                                            entities  favorited  retweeted  \\\n",
       "0  {'hashtags': [], 'symbols': [], 'user_mentions...      False      False   \n",
       "1  {'hashtags': [], 'symbols': [], 'user_mentions...      False      False   \n",
       "2  {'hashtags': [], 'symbols': [], 'user_mentions...      False      False   \n",
       "3  {'hashtags': [], 'symbols': [], 'user_mentions...      False      False   \n",
       "4  {'hashtags': [], 'symbols': [], 'user_mentions...      False      False   \n",
       "\n",
       "  lang  \n",
       "0   en  \n",
       "1   en  \n",
       "2   en  \n",
       "3   en  \n",
       "4   en  "
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_csv(\"Economy_USA_final.csv\")\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "16799"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>created_at</th>\n",
       "      <th>id</th>\n",
       "      <th>full_text</th>\n",
       "      <th>source</th>\n",
       "      <th>truncated</th>\n",
       "      <th>metadata</th>\n",
       "      <th>in_reply_to_status_id</th>\n",
       "      <th>in_reply_to_user_id</th>\n",
       "      <th>in_reply_to_screen_name</th>\n",
       "      <th>user</th>\n",
       "      <th>place</th>\n",
       "      <th>is_quote_status</th>\n",
       "      <th>retweet_count</th>\n",
       "      <th>favorite_count</th>\n",
       "      <th>entities</th>\n",
       "      <th>favorited</th>\n",
       "      <th>retweeted</th>\n",
       "      <th>lang</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>2020-04-29 18:05:09</td>\n",
       "      <td>1255558756890374144</td>\n",
       "      <td>“The Federal Reserve said the U.S. economy has...</td>\n",
       "      <td>Twitter for iPhone</td>\n",
       "      <td>False</td>\n",
       "      <td>{'iso_language_code': 'en', 'result_type': 're...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{'id': 940032448057896960, 'id_str': '94003244...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>{'hashtags': [], 'symbols': [], 'user_mentions...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>en</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0           created_at                   id  \\\n",
       "4           0  2020-04-29 18:05:09  1255558756890374144   \n",
       "\n",
       "                                           full_text              source  \\\n",
       "4  “The Federal Reserve said the U.S. economy has...  Twitter for iPhone   \n",
       "\n",
       "   truncated                                           metadata  \\\n",
       "4      False  {'iso_language_code': 'en', 'result_type': 're...   \n",
       "\n",
       "   in_reply_to_status_id  in_reply_to_user_id in_reply_to_screen_name  \\\n",
       "4                    NaN                  NaN                     NaN   \n",
       "\n",
       "                                                user place  is_quote_status  \\\n",
       "4  {'id': 940032448057896960, 'id_str': '94003244...   NaN            False   \n",
       "\n",
       "   retweet_count  favorite_count  \\\n",
       "4              0               1   \n",
       "\n",
       "                                            entities  favorited  retweeted  \\\n",
       "4  {'hashtags': [], 'symbols': [], 'user_mentions...      False      False   \n",
       "\n",
       "  lang  \n",
       "4   en  "
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[data['id']==1255558756890374144]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>created_at</th>\n",
       "      <th>id</th>\n",
       "      <th>full_text</th>\n",
       "      <th>source</th>\n",
       "      <th>truncated</th>\n",
       "      <th>metadata</th>\n",
       "      <th>in_reply_to_status_id</th>\n",
       "      <th>in_reply_to_user_id</th>\n",
       "      <th>in_reply_to_screen_name</th>\n",
       "      <th>user</th>\n",
       "      <th>place</th>\n",
       "      <th>is_quote_status</th>\n",
       "      <th>retweet_count</th>\n",
       "      <th>favorite_count</th>\n",
       "      <th>entities</th>\n",
       "      <th>favorited</th>\n",
       "      <th>retweeted</th>\n",
       "      <th>lang</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>14500</th>\n",
       "      <td>0</td>\n",
       "      <td>2020-04-28 08:37:05</td>\n",
       "      <td>1255053411667578881</td>\n",
       "      <td>RT @RURALINDIA: Movie : The anatomy of NPAs : ...</td>\n",
       "      <td>Twitter for Android</td>\n",
       "      <td>False</td>\n",
       "      <td>{'iso_language_code': 'en', 'result_type': 're...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{'id': 973883327353532416, 'id_str': '97388332...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>384</td>\n",
       "      <td>0</td>\n",
       "      <td>{'hashtags': [], 'symbols': [], 'user_mentions...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>en</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14501</th>\n",
       "      <td>0</td>\n",
       "      <td>2020-04-28 08:32:36</td>\n",
       "      <td>1255052283097886720</td>\n",
       "      <td>RT @OANN: San Diego residents demand reopening...</td>\n",
       "      <td>Twitter for iPad</td>\n",
       "      <td>False</td>\n",
       "      <td>{'iso_language_code': 'en', 'result_type': 're...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{'id': 744950188557615104, 'id_str': '74495018...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>529</td>\n",
       "      <td>0</td>\n",
       "      <td>{'hashtags': [{'text': 'OANN', 'indices': [89,...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>en</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14502</th>\n",
       "      <td>0</td>\n",
       "      <td>2020-04-28 08:32:31</td>\n",
       "      <td>1255052260033216514</td>\n",
       "      <td>RT @BobPersis: @RepAdamSchiff 90% of COVID dea...</td>\n",
       "      <td>Twitter for Android</td>\n",
       "      <td>False</td>\n",
       "      <td>{'iso_language_code': 'en', 'result_type': 're...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{'id': 1090858461812801537, 'id_str': '1090858...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>205</td>\n",
       "      <td>0</td>\n",
       "      <td>{'hashtags': [], 'symbols': [], 'user_mentions...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>en</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14503</th>\n",
       "      <td>0</td>\n",
       "      <td>2020-04-28 08:32:18</td>\n",
       "      <td>1255052205503074306</td>\n",
       "      <td>RT @stephphilip8: \"'YOU'RE GOING TO SEE THE EC...</td>\n",
       "      <td>Twitter Web App</td>\n",
       "      <td>False</td>\n",
       "      <td>{'iso_language_code': 'en', 'result_type': 're...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{'id': 3308193378, 'id_str': '3308193378', 'na...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>129</td>\n",
       "      <td>0</td>\n",
       "      <td>{'hashtags': [], 'symbols': [], 'user_mentions...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>en</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14504</th>\n",
       "      <td>0</td>\n",
       "      <td>2020-04-28 08:30:28</td>\n",
       "      <td>1255051746323333120</td>\n",
       "      <td>if u know anything about argentina’s economy c...</td>\n",
       "      <td>Twitter for iPhone</td>\n",
       "      <td>False</td>\n",
       "      <td>{'iso_language_code': 'en', 'result_type': 're...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{'id': 1075719961795006464, 'id_str': '1075719...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>{'hashtags': [], 'symbols': [], 'user_mentions...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>en</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14505</th>\n",
       "      <td>0</td>\n",
       "      <td>2020-04-28 08:30:05</td>\n",
       "      <td>1255051648822661121</td>\n",
       "      <td>RT @stengel: When Trump says we had the greate...</td>\n",
       "      <td>BizCreditCo</td>\n",
       "      <td>False</td>\n",
       "      <td>{'iso_language_code': 'en', 'result_type': 're...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{'id': 895420635878498304, 'id_str': '89542063...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>130</td>\n",
       "      <td>0</td>\n",
       "      <td>{'hashtags': [], 'symbols': [], 'user_mentions...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>en</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14506</th>\n",
       "      <td>0</td>\n",
       "      <td>2020-04-28 08:29:59</td>\n",
       "      <td>1255051623749095425</td>\n",
       "      <td>RT @dkgochhayat: Thank u @nsitharaman, @ianura...</td>\n",
       "      <td>Twitter for iPhone</td>\n",
       "      <td>False</td>\n",
       "      <td>{'iso_language_code': 'en', 'result_type': 're...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{'id': 812622236, 'id_str': '812622236', 'name...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>{'hashtags': [], 'symbols': [], 'user_mentions...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>en</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14507</th>\n",
       "      <td>0</td>\n",
       "      <td>2020-04-28 08:29:41</td>\n",
       "      <td>1255051546259337218</td>\n",
       "      <td>@DrMannySenate you doctors who have used this ...</td>\n",
       "      <td>Twitter Web App</td>\n",
       "      <td>False</td>\n",
       "      <td>{'iso_language_code': 'en', 'result_type': 're...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.135626e+18</td>\n",
       "      <td>DrMannySenate</td>\n",
       "      <td>{'id': 1482751176, 'id_str': '1482751176', 'na...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>{'hashtags': [], 'symbols': [], 'user_mentions...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>en</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14508</th>\n",
       "      <td>0</td>\n",
       "      <td>2020-04-28 08:29:40</td>\n",
       "      <td>1255051542232793090</td>\n",
       "      <td>RT @Tact_Investor: US vs China war will Destro...</td>\n",
       "      <td>Twitter Web App</td>\n",
       "      <td>False</td>\n",
       "      <td>{'iso_language_code': 'en', 'result_type': 're...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{'id': 1174230941725859840, 'id_str': '1174230...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>{'hashtags': [], 'symbols': [], 'user_mentions...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>en</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14509</th>\n",
       "      <td>0</td>\n",
       "      <td>2020-04-28 08:29:20</td>\n",
       "      <td>1255051460716498945</td>\n",
       "      <td>RT @BVMsDaughter: We actually stayed home for ...</td>\n",
       "      <td>Twitter for Android</td>\n",
       "      <td>False</td>\n",
       "      <td>{'iso_language_code': 'en', 'result_type': 're...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>{'id': 1120800343963709441, 'id_str': '1120800...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>3986</td>\n",
       "      <td>0</td>\n",
       "      <td>{'hashtags': [], 'symbols': [], 'user_mentions...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>en</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       Unnamed: 0           created_at                   id  \\\n",
       "14500           0  2020-04-28 08:37:05  1255053411667578881   \n",
       "14501           0  2020-04-28 08:32:36  1255052283097886720   \n",
       "14502           0  2020-04-28 08:32:31  1255052260033216514   \n",
       "14503           0  2020-04-28 08:32:18  1255052205503074306   \n",
       "14504           0  2020-04-28 08:30:28  1255051746323333120   \n",
       "14505           0  2020-04-28 08:30:05  1255051648822661121   \n",
       "14506           0  2020-04-28 08:29:59  1255051623749095425   \n",
       "14507           0  2020-04-28 08:29:41  1255051546259337218   \n",
       "14508           0  2020-04-28 08:29:40  1255051542232793090   \n",
       "14509           0  2020-04-28 08:29:20  1255051460716498945   \n",
       "\n",
       "                                               full_text               source  \\\n",
       "14500  RT @RURALINDIA: Movie : The anatomy of NPAs : ...  Twitter for Android   \n",
       "14501  RT @OANN: San Diego residents demand reopening...     Twitter for iPad   \n",
       "14502  RT @BobPersis: @RepAdamSchiff 90% of COVID dea...  Twitter for Android   \n",
       "14503  RT @stephphilip8: \"'YOU'RE GOING TO SEE THE EC...      Twitter Web App   \n",
       "14504  if u know anything about argentina’s economy c...   Twitter for iPhone   \n",
       "14505  RT @stengel: When Trump says we had the greate...          BizCreditCo   \n",
       "14506  RT @dkgochhayat: Thank u @nsitharaman, @ianura...   Twitter for iPhone   \n",
       "14507  @DrMannySenate you doctors who have used this ...      Twitter Web App   \n",
       "14508  RT @Tact_Investor: US vs China war will Destro...      Twitter Web App   \n",
       "14509  RT @BVMsDaughter: We actually stayed home for ...  Twitter for Android   \n",
       "\n",
       "       truncated                                           metadata  \\\n",
       "14500      False  {'iso_language_code': 'en', 'result_type': 're...   \n",
       "14501      False  {'iso_language_code': 'en', 'result_type': 're...   \n",
       "14502      False  {'iso_language_code': 'en', 'result_type': 're...   \n",
       "14503      False  {'iso_language_code': 'en', 'result_type': 're...   \n",
       "14504      False  {'iso_language_code': 'en', 'result_type': 're...   \n",
       "14505      False  {'iso_language_code': 'en', 'result_type': 're...   \n",
       "14506      False  {'iso_language_code': 'en', 'result_type': 're...   \n",
       "14507      False  {'iso_language_code': 'en', 'result_type': 're...   \n",
       "14508      False  {'iso_language_code': 'en', 'result_type': 're...   \n",
       "14509      False  {'iso_language_code': 'en', 'result_type': 're...   \n",
       "\n",
       "       in_reply_to_status_id  in_reply_to_user_id in_reply_to_screen_name  \\\n",
       "14500                    NaN                  NaN                     NaN   \n",
       "14501                    NaN                  NaN                     NaN   \n",
       "14502                    NaN                  NaN                     NaN   \n",
       "14503                    NaN                  NaN                     NaN   \n",
       "14504                    NaN                  NaN                     NaN   \n",
       "14505                    NaN                  NaN                     NaN   \n",
       "14506                    NaN                  NaN                     NaN   \n",
       "14507                    NaN         1.135626e+18           DrMannySenate   \n",
       "14508                    NaN                  NaN                     NaN   \n",
       "14509                    NaN                  NaN                     NaN   \n",
       "\n",
       "                                                    user place  \\\n",
       "14500  {'id': 973883327353532416, 'id_str': '97388332...   NaN   \n",
       "14501  {'id': 744950188557615104, 'id_str': '74495018...   NaN   \n",
       "14502  {'id': 1090858461812801537, 'id_str': '1090858...   NaN   \n",
       "14503  {'id': 3308193378, 'id_str': '3308193378', 'na...   NaN   \n",
       "14504  {'id': 1075719961795006464, 'id_str': '1075719...   NaN   \n",
       "14505  {'id': 895420635878498304, 'id_str': '89542063...   NaN   \n",
       "14506  {'id': 812622236, 'id_str': '812622236', 'name...   NaN   \n",
       "14507  {'id': 1482751176, 'id_str': '1482751176', 'na...   NaN   \n",
       "14508  {'id': 1174230941725859840, 'id_str': '1174230...   NaN   \n",
       "14509  {'id': 1120800343963709441, 'id_str': '1120800...   NaN   \n",
       "\n",
       "       is_quote_status  retweet_count  favorite_count  \\\n",
       "14500            False            384               0   \n",
       "14501            False            529               0   \n",
       "14502            False            205               0   \n",
       "14503            False            129               0   \n",
       "14504            False              0               2   \n",
       "14505            False            130               0   \n",
       "14506            False              1               0   \n",
       "14507            False              0               0   \n",
       "14508            False              1               0   \n",
       "14509            False           3986               0   \n",
       "\n",
       "                                                entities  favorited  \\\n",
       "14500  {'hashtags': [], 'symbols': [], 'user_mentions...      False   \n",
       "14501  {'hashtags': [{'text': 'OANN', 'indices': [89,...      False   \n",
       "14502  {'hashtags': [], 'symbols': [], 'user_mentions...      False   \n",
       "14503  {'hashtags': [], 'symbols': [], 'user_mentions...      False   \n",
       "14504  {'hashtags': [], 'symbols': [], 'user_mentions...      False   \n",
       "14505  {'hashtags': [], 'symbols': [], 'user_mentions...      False   \n",
       "14506  {'hashtags': [], 'symbols': [], 'user_mentions...      False   \n",
       "14507  {'hashtags': [], 'symbols': [], 'user_mentions...      False   \n",
       "14508  {'hashtags': [], 'symbols': [], 'user_mentions...      False   \n",
       "14509  {'hashtags': [], 'symbols': [], 'user_mentions...      False   \n",
       "\n",
       "       retweeted lang  \n",
       "14500      False   en  \n",
       "14501      False   en  \n",
       "14502      False   en  \n",
       "14503      False   en  \n",
       "14504      False   en  \n",
       "14505      False   en  \n",
       "14506      False   en  \n",
       "14507      False   en  \n",
       "14508      False   en  \n",
       "14509      False   en  "
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[14500:14510]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'_id': ObjectId('5ea9c4c217dfdac82a91bc1a'), 'id': 1255051546259337218, 'user_name': 'Alva', 'user_id': '1482751176', 'content': '@DrMannySenate you doctors who have used this virus to destroy U S jobs, economy &amp; business and violate Americans Constitutional RightsCivil Liberty’s lawyer you for lawsuits coming your way for destroying American livelihoods! It’s coming you all way!', 'created_at': '2020-04-28 08:29:41', 'location': '', 'hashtags': [], 'mentions': ['1135625824282058753'], 'in_reply_to_user_id': 1135625824282058753, 'in_reply_to_status_id': None, 'retweedt_orid_tweetID': -1, 'retweetedFrom_id': -1, 'retweetedFrom_name': 'NA', 'media': [], 'retweet': False, 'FavCount': 0, 'Orig_retweet_fav': -1, 'source': 'Twitter Web App', 'retweet_count': 0, 'lang': 'en'}\n"
     ]
    }
   ],
   "source": [
    "myquery = {'id': 1255051546259337218}\n",
    "\n",
    "mydoc = my_tweets_table.find(myquery)\n",
    "\n",
    "for x in mydoc:\n",
    "    print(x)\n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "16898\n"
     ]
    }
   ],
   "source": [
    "myquery = {}\n",
    "\n",
    "mydoc = my_tweets_table.find(myquery)\n",
    "count = 0\n",
    "for x in mydoc:\n",
    "    count = count + 1\n",
    "print(count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}