Twitter.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Twitter.ipynb",
      "provenance": [],
      "toc_visible": true,
      "authorship_tag": "ABX9TyM0nKHl3z8DORj3iJyJ2kNS",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "24c4c16d7b2f4d1f81db9c46eaecc402": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DropdownModel",
          "model_module_version": "1.5.0",
          "state": {
            "_options_labels": [
              "nan",
              "US",
              "PK",
              "CA",
              "IN",
              "PH",
              "TH",
              "CO",
              "NZ",
              "ZA",
              "ES",
              "ID",
              "SV",
              "MX",
              "AU",
              "MY",
              "IT",
              "BR",
              "JP",
              "PE",
              "SG",
              "EC",
              "FR",
              "PA",
              "HR",
              "AR",
              "MQ",
              "TW",
              "UG",
              "KW",
              "MM",
              "CH",
              "ZW",
              "CI",
              "AE",
              "NP",
              "BO",
              "NL",
              "TZ",
              "PL",
              "SA",
              "DE",
              "JM",
              "MW",
              "GB",
              "CL",
              "IL",
              "TR",
              "MV",
              "MT",
              "LK",
              "LI",
              "RW",
              "BD",
              "RU",
              "YE",
              "IE",
              "FI",
              "AO",
              "DK",
              "AT",
              "QA",
              "KE",
              "CD",
              "KR",
              "BW",
              "FJ",
              "TG",
              "TT",
              "RS",
              "KZ",
              "BE",
              "RE",
              "PT",
              "SL",
              "GH",
              "MN",
              "CN",
              "BY",
              "CU",
              "VE",
              "NO",
              "LY",
              "LV",
              "DO",
              "SE",
              "TD",
              "TL",
              "CZ",
              "UY",
              "PY",
              "UA",
              "GP",
              "MK",
              "VN",
              "SN",
              "MA",
              "BS",
              "CY",
              "GN",
              "ML",
              "LS",
              "LU",
              "CR",
              "PR",
              "GR",
              "MZ",
              "BJ",
              "DZ",
              "HN",
              "AL",
              "ET",
              "LB",
              "TN",
              "GF",
              "KH",
              "GT",
              "NI",
              "HT",
              "IR",
              "LC",
              "SK",
              "LR"
            ],
            "_view_name": "DropdownView",
            "style": "IPY_MODEL_6b55d958f2214f09b06709e17640e5d9",
            "_dom_classes": [],
            "description": "",
            "_model_name": "DropdownModel",
            "index": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "disabled": false,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b5b7f6d508304befa0379dd4556a1001"
          }
        },
        "6b55d958f2214f09b06709e17640e5d9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b5b7f6d508304befa0379dd4556a1001": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/yliu9418/LeetCode-Go/blob/master/Twitter.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZaG_IgpqQzhG"
      },
      "source": [
        "import gzip\n",
        "import shutil\n",
        "import os\n",
        "import csv\n",
        "import linecache\n",
        "from shutil import copyfile\n",
        "import ipywidgets as widgets\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import wget\n",
        "\n",
        "CONSUMER_KEY = \"LJ5ugdnBADLYRHzuPlI16rzMF\"\n",
        "CONSUMER_SECRET_KEY = \"Fj4XtBamfNyP1JE5G5wAPsErJ6y4F3MTlrA5wJQ9J0NsyRgSPr\"\n",
        "ACCESS_TOKEN_KEY = \"890342461553422336-hRSVKVUh9wxYfBTCyHJsWANZDX2ATGF\"\n",
        "ACCESS_TOKEN_SECRET_KEY= \"OTMlIyuk7pfcQgO64XwmeQPx8Me2A8NKnPiGyAgECuMdi\"\n",
        "\n",
        "# dataset_URL = \"https://github.com/thepanacealab/covid19_twitter/blob/master/dailies/2020-11-01/2020-11-01_clean-dataset.tsv.gz?raw=true\" #@param {type:\"string\"}\n",
        "dataset_URL = \"https://github.com/thepanacealab/covid19_twitter/blob/master/dailies/2021-08-01/2021-08-01_clean-dataset.tsv.gz?raw=true\"\n",
        "#Downloads the dataset (compressed in a GZ format)\n",
        "#!wget dataset_URL -O clean-dataset.tsv.gz\n",
        "wget.download(dataset_URL, out='clean-dataset.tsv.gz')\n",
        "\n",
        "#Unzips the dataset and gets the TSV dataset\n",
        "with gzip.open('clean-dataset.tsv.gz', 'rb') as f_in:\n",
        "    with open('clean-dataset.tsv', 'wb') as f_out:\n",
        "        shutil.copyfileobj(f_in, f_out)\n",
        "\n",
        "#Deletes the compressed GZ file\n",
        "os.unlink(\"clean-dataset.tsv.gz\")\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 49,
          "referenced_widgets": [
            "24c4c16d7b2f4d1f81db9c46eaecc402",
            "6b55d958f2214f09b06709e17640e5d9",
            "b5b7f6d508304befa0379dd4556a1001"
          ]
        },
        "id": "4zr6Gfq6Q7Wv",
        "outputId": "72bc4daf-ff37-4d5a-de1d-5dddc3398f7a"
      },
      "source": [
        "#Gets all possible languages from the dataset\n",
        "df = pd.read_csv('clean-dataset.tsv',sep=\"\\t\")\n",
        "lang_list = df.lang.unique()\n",
        "lang_list= sorted(np.append(lang_list,'all'))\n",
        "lang_picker = widgets.Dropdown(options=lang_list, value=\"en\")\n",
        "lang_picker\n",
        "\n",
        "#Gets all possible countries from the dataset\n",
        "#df = pd.read_csv('clean-dataset.tsv',sep=\"\\t\")\n",
        "country_list = df.country_code.unique()\n",
        "#country_list= sorted(np.append(country_list,'all'))\n",
        "country_picker = widgets.Dropdown(options=country_list, value=\"US\")\n",
        "country_picker"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "24c4c16d7b2f4d1f81db9c46eaecc402",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Dropdown(index=1, options=(nan, 'US', 'PK', 'CA', 'IN', 'PH', 'TH', 'CO', 'NZ', 'ZA', 'ES', 'ID', 'SV', 'MX', …"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8g_cwBaIRBSM"
      },
      "source": [
        "#Creates a new clean dataset with the specified language (if specified)\n",
        "filtered_language = lang_picker.value\n",
        "filtered_country = country_picker.value\n",
        "\n",
        "#If no language specified, it will get all records from the dataset\n",
        "if filtered_language == \"\":\n",
        "  copyfile('clean-dataset.tsv', 'clean-dataset-filtered.tsv')\n",
        "\n",
        "#If language specified, it will create another tsv file with the filtered records\n",
        "else:\n",
        "  filtered_tw = list()\n",
        "  current_line = 1\n",
        "  with open(\"clean-dataset.tsv\") as tsvfile:\n",
        "    tsvreader = csv.reader(tsvfile, delimiter=\"\\t\")\n",
        "\n",
        "    if current_line == 1:\n",
        "      filtered_tw.append(linecache.getline(\"clean-dataset.tsv\", current_line))\n",
        "\n",
        "      for line in tsvreader:\n",
        "        if line[3] == filtered_language and line[4] == filtered_country:\n",
        "          filtered_tw.append(linecache.getline(\"clean-dataset.tsv\", current_line))\n",
        "        current_line += 1\n",
        "\n",
        "  print('\\033[1mShowing first 5 tweets from the filtered dataset\\033[0m')\n",
        "  print(filtered_tw[1:(6 if len(filtered_tw) > 6 else len(filtered_tw))])\n",
        "\n",
        "  with open('clean-dataset-filtered.tsv', 'w') as f_output:\n",
        "      for item in filtered_tw:\n",
        "          f_output.write(item)\n",
        "\n",
        "\n",
        "import json\n",
        "import tweepy\n",
        "from tweepy import OAuthHandler\n",
        "\n",
        "\n",
        "#Creates a JSON Files with the API credentials\n",
        "with open('api_keys.json', 'w') as outfile:\n",
        "    json.dump({\n",
        "    \"consumer_key\":CONSUMER_KEY,\n",
        "    \"consumer_secret\":CONSUMER_SECRET_KEY,\n",
        "    \"access_token\":ACCESS_TOKEN_KEY,\n",
        "    \"access_token_secret\": ACCESS_TOKEN_SECRET_KEY\n",
        "     }, outfile)\n",
        "    \n",
        "\n",
        "from IPython.display import clear_output\n",
        "\n",
        "wget.download(\"https://raw.githubusercontent.com/thepanacealab/SMMT/master/data_acquisition/get_metadata.py\", \"get_metadata.py\")\n",
        "    \n",
        "clear_output()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FO0mZjqXREZd"
      },
      "source": [
        "!python get_metadata.py -i clean-dataset-filtered.tsv -o hydrated_tweets -k api_keys.json -m e\n",
        "\n",
        "clear_output()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xV-rgv0tRFGZ"
      },
      "source": [
        "from IPython.display import clear_output\n",
        "\n",
        "!wget https://raw.githubusercontent.com/thepanacealab/SMMT/master/data_preprocessing/parse_json_lite.py -O parse_json_lite.py\n",
        "!wget https://raw.githubusercontent.com/thepanacealab/SMMT/master/data_preprocessing/fields.py -O fields.py\n",
        "\n",
        "!pip install emot --upgrade\n",
        "!pip install emoji --upgrade\n",
        "\n",
        "clear_output()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ANK9O6omRHhu"
      },
      "source": [
        "import random \n",
        "\n",
        "no_samples = \"1000\" #@param {type:\"string\"}\n",
        "list_tweets = None\n",
        "\n",
        "with open(\"hydrated_tweets_short.json\", \"r\") as myfile:\n",
        "    list_tweets = list(myfile)\n",
        "\n",
        "if int(no_samples) > len(list_tweets):\n",
        "    no_samples = len(list_tweets)\n",
        "\n",
        "sample = random.sample(list_tweets, int(no_samples))\n",
        "\n",
        "file = open(\"sample_data.json\", \"w\")\n",
        "for i in sample:\n",
        "  file.write(i)\n",
        "file.close() #This close() is important"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}