data_wrangling.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "#import nltk\n",
    "from matplotlib import pyplot as plt\n",
    "from scrape_data import *\n",
    "from process_text import *\n",
    "from helper import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Initialize the dict to store all text lists for below titles\n",
    "text_lists = {}\n",
    "titles = ['Data Scientist', 'Machine Learning Engineer', 'Data Engineer']\n",
    "# Grab the tokens list and store them in the dict\n",
    "for title in titles:\n",
    "    text_lists[title] = plot_profile(title=title, first_n_postings=120, return_text_list=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Make the dict of skills to investigate\n",
    "\n",
    "languages = ['Python', 'R', 'SQL', 'Java', 'C', 'C++', 'C#', 'Scala', 'Perl', 'Julia', \n",
    "             'Javascript', 'HTML', 'CSS', 'PHP', 'Ruby', 'Lua', 'MATLAB', 'SAS'] \n",
    "\n",
    "big_data = ['Hadoop', 'MapReduce', 'Hive', 'Pig', 'Cascading', 'Scalding', 'Cascalog', 'HBase', 'Sqoop', \n",
    "            'Mahout', 'Oozie', 'Flume', 'ZooKeeper', 'Spark', 'Storm', 'Shark', 'Impala', 'Elasticsearch', \n",
    "            'Kafka', 'Flink', 'Kinesis', 'Presto', 'Hume', 'Airflow', 'Azkabhan', 'Luigi', 'Cassandra']\n",
    "\n",
    "dl = ['TensorFlow', 'Keras', 'PyTorch', 'Theano', 'Deeplearning4J', 'Caffe', 'TFLearn', 'Torch', \n",
    "      'OpenCV', 'MXNet', 'Microsoft Cognitive Toolkit', 'Lasagne']\n",
    "\n",
    "cloud = ['AWS', 'GCP', 'Azure']\n",
    "\n",
    "ml = ['Natural Language Processing', 'Computer Vision', 'Speech Recognition', 'Fraud Detection',\n",
    "      'Recommender System', 'Image Recognition', 'Object Dectection', 'Chatbot',  'Sentiment Analysis']\n",
    "\n",
    "visualization = ['Dimple', 'D3.js', 'Ggplot', 'Shiny', 'Plotly', 'Matplotlib', 'Seaborn', \n",
    "                'Bokeh', 'Tableau']\n",
    "\n",
    "other = ['Pandas', 'Numpy', 'Scipy', 'Sklearn', 'Scikit-Learn', 'Docker', 'Git', 'Jira', 'Kaggle']\n",
    "\n",
    "dict_to_check = {'Programming Languages': languages,\n",
    "                 'Big Data Technologies': big_data,\n",
    "                 'Deep Learning Frameworks': dl,\n",
    "                 'Cloud Computing Platforms': cloud,\n",
    "                 'Machine Learning Application': ml,\n",
    "                 'Visualization Tools': visualization,\n",
    "                 'Other': other}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check the frequency and store in dict\n",
    "freq_dict = {}\n",
    "for title in text_lists.keys():\n",
    "    freq_dict[title] = check_freq(dict_to_check=dict_to_check, text_list=text_lists[title])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>Python</th>\n",
       "      <th>R</th>\n",
       "      <th>SQL</th>\n",
       "      <th>Java</th>\n",
       "      <th>C</th>\n",
       "      <th>C++</th>\n",
       "      <th>C#</th>\n",
       "      <th>Scala</th>\n",
       "      <th>Perl</th>\n",
       "      <th>Julia</th>\n",
       "      <th>...</th>\n",
       "      <th>Tableau</th>\n",
       "      <th>Pandas</th>\n",
       "      <th>Numpy</th>\n",
       "      <th>Scipy</th>\n",
       "      <th>Sklearn</th>\n",
       "      <th>Scikit-Learn</th>\n",
       "      <th>Docker</th>\n",
       "      <th>Git</th>\n",
       "      <th>Jira</th>\n",
       "      <th>Kaggle</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">Data Engineer</th>\n",
       "      <th>Big Data Technologies</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Cloud Computing Platforms</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Deep Learning Frameworks</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Machine Learning Application</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Other</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 87 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            Python   R  SQL  Java   C  C++  \\\n",
       "Data Engineer Big Data Technologies            NaN NaN  NaN   NaN NaN  NaN   \n",
       "              Cloud Computing Platforms        NaN NaN  NaN   NaN NaN  NaN   \n",
       "              Deep Learning Frameworks         NaN NaN  NaN   NaN NaN  NaN   \n",
       "              Machine Learning Application     NaN NaN  NaN   NaN NaN  NaN   \n",
       "              Other                            NaN NaN  NaN   NaN NaN  NaN   \n",
       "\n",
       "                                            C#  Scala  Perl  Julia   ...    \\\n",
       "Data Engineer Big Data Technologies        NaN    NaN   NaN    NaN   ...     \n",
       "              Cloud Computing Platforms    NaN    NaN   NaN    NaN   ...     \n",
       "              Deep Learning Frameworks     NaN    NaN   NaN    NaN   ...     \n",
       "              Machine Learning Application NaN    NaN   NaN    NaN   ...     \n",
       "              Other                        NaN    NaN   NaN    NaN   ...     \n",
       "\n",
       "                                            Tableau  Pandas  Numpy  Scipy  \\\n",
       "Data Engineer Big Data Technologies             NaN     NaN    NaN    NaN   \n",
       "              Cloud Computing Platforms         NaN     NaN    NaN    NaN   \n",
       "              Deep Learning Frameworks          NaN     NaN    NaN    NaN   \n",
       "              Machine Learning Application      NaN     NaN    NaN    NaN   \n",
       "              Other                             NaN     3.0    0.0    2.0   \n",
       "\n",
       "                                            Sklearn  Scikit-Learn  Docker  \\\n",
       "Data Engineer Big Data Technologies             NaN           NaN     NaN   \n",
       "              Cloud Computing Platforms         NaN           NaN     NaN   \n",
       "              Deep Learning Frameworks          NaN           NaN     NaN   \n",
       "              Machine Learning Application      NaN           NaN     NaN   \n",
       "              Other                             0.0           0.0     8.0   \n",
       "\n",
       "                                             Git  Jira  Kaggle  \n",
       "Data Engineer Big Data Technologies          NaN   NaN     NaN  \n",
       "              Cloud Computing Platforms      NaN   NaN     NaN  \n",
       "              Deep Learning Frameworks       NaN   NaN     NaN  \n",
       "              Machine Learning Application   NaN   NaN     NaN  \n",
       "              Other                         29.0   1.0     0.0  \n",
       "\n",
       "[5 rows x 87 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Convert the dict to a pandas df\n",
    "df = pd.DataFrame.from_dict({(i,j): freq_dict[i][j] \n",
    "                             for i in freq_dict.keys()\n",
    "                             for j in freq_dict[i].keys()},\n",
    "                            orient='index')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>level_0</th>\n",
       "      <th>level_1</th>\n",
       "      <th>Python</th>\n",
       "      <th>R</th>\n",
       "      <th>SQL</th>\n",
       "      <th>Java</th>\n",
       "      <th>C</th>\n",
       "      <th>C++</th>\n",
       "      <th>C#</th>\n",
       "      <th>Scala</th>\n",
       "      <th>...</th>\n",
       "      <th>Tableau</th>\n",
       "      <th>Pandas</th>\n",
       "      <th>Numpy</th>\n",
       "      <th>Scipy</th>\n",
       "      <th>Sklearn</th>\n",
       "      <th>Scikit-Learn</th>\n",
       "      <th>Docker</th>\n",
       "      <th>Git</th>\n",
       "      <th>Jira</th>\n",
       "      <th>Kaggle</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Data Engineer</td>\n",
       "      <td>Big Data Technologies</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Data Engineer</td>\n",
       "      <td>Cloud Computing Platforms</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Data Engineer</td>\n",
       "      <td>Deep Learning Frameworks</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Data Engineer</td>\n",
       "      <td>Machine Learning Application</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Data Engineer</td>\n",
       "      <td>Other</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 89 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         level_0                       level_1  Python   R  SQL  Java   C  \\\n",
       "0  Data Engineer         Big Data Technologies     NaN NaN  NaN   NaN NaN   \n",
       "1  Data Engineer     Cloud Computing Platforms     NaN NaN  NaN   NaN NaN   \n",
       "2  Data Engineer      Deep Learning Frameworks     NaN NaN  NaN   NaN NaN   \n",
       "3  Data Engineer  Machine Learning Application     NaN NaN  NaN   NaN NaN   \n",
       "4  Data Engineer                         Other     NaN NaN  NaN   NaN NaN   \n",
       "\n",
       "   C++  C#  Scala   ...    Tableau  Pandas  Numpy  Scipy  Sklearn  \\\n",
       "0  NaN NaN    NaN   ...        NaN     NaN    NaN    NaN      NaN   \n",
       "1  NaN NaN    NaN   ...        NaN     NaN    NaN    NaN      NaN   \n",
       "2  NaN NaN    NaN   ...        NaN     NaN    NaN    NaN      NaN   \n",
       "3  NaN NaN    NaN   ...        NaN     NaN    NaN    NaN      NaN   \n",
       "4  NaN NaN    NaN   ...        NaN     3.0    0.0    2.0      0.0   \n",
       "\n",
       "   Scikit-Learn  Docker   Git  Jira  Kaggle  \n",
       "0           NaN     NaN   NaN   NaN     NaN  \n",
       "1           NaN     NaN   NaN   NaN     NaN  \n",
       "2           NaN     NaN   NaN   NaN     NaN  \n",
       "3           NaN     NaN   NaN   NaN     NaN  \n",
       "4           0.0     8.0  29.0   1.0     0.0  \n",
       "\n",
       "[5 rows x 89 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Reset the index to include both title and category as columns\n",
    "df = df.reset_index()\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>category</th>\n",
       "      <th>Python</th>\n",
       "      <th>R</th>\n",
       "      <th>SQL</th>\n",
       "      <th>Java</th>\n",
       "      <th>C</th>\n",
       "      <th>C++</th>\n",
       "      <th>C#</th>\n",
       "      <th>Scala</th>\n",
       "      <th>...</th>\n",
       "      <th>Tableau</th>\n",
       "      <th>Pandas</th>\n",
       "      <th>Numpy</th>\n",
       "      <th>Scipy</th>\n",
       "      <th>Sklearn</th>\n",
       "      <th>Scikit-Learn</th>\n",
       "      <th>Docker</th>\n",
       "      <th>Git</th>\n",
       "      <th>Jira</th>\n",
       "      <th>Kaggle</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Data Engineer</td>\n",
       "      <td>Big Data Technologies</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Data Engineer</td>\n",
       "      <td>Cloud Computing Platforms</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Data Engineer</td>\n",
       "      <td>Deep Learning Frameworks</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Data Engineer</td>\n",
       "      <td>Machine Learning Application</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Data Engineer</td>\n",
       "      <td>Other</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 89 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           title                      category  Python   R  SQL  Java   C  \\\n",
       "0  Data Engineer         Big Data Technologies     NaN NaN  NaN   NaN NaN   \n",
       "1  Data Engineer     Cloud Computing Platforms     NaN NaN  NaN   NaN NaN   \n",
       "2  Data Engineer      Deep Learning Frameworks     NaN NaN  NaN   NaN NaN   \n",
       "3  Data Engineer  Machine Learning Application     NaN NaN  NaN   NaN NaN   \n",
       "4  Data Engineer                         Other     NaN NaN  NaN   NaN NaN   \n",
       "\n",
       "   C++  C#  Scala   ...    Tableau  Pandas  Numpy  Scipy  Sklearn  \\\n",
       "0  NaN NaN    NaN   ...        NaN     NaN    NaN    NaN      NaN   \n",
       "1  NaN NaN    NaN   ...        NaN     NaN    NaN    NaN      NaN   \n",
       "2  NaN NaN    NaN   ...        NaN     NaN    NaN    NaN      NaN   \n",
       "3  NaN NaN    NaN   ...        NaN     NaN    NaN    NaN      NaN   \n",
       "4  NaN NaN    NaN   ...        NaN     3.0    0.0    2.0      0.0   \n",
       "\n",
       "   Scikit-Learn  Docker   Git  Jira  Kaggle  \n",
       "0           NaN     NaN   NaN   NaN     NaN  \n",
       "1           NaN     NaN   NaN   NaN     NaN  \n",
       "2           NaN     NaN   NaN   NaN     NaN  \n",
       "3           NaN     NaN   NaN   NaN     NaN  \n",
       "4           0.0     8.0  29.0   1.0     0.0  \n",
       "\n",
       "[5 rows x 89 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Rename the first two columns\n",
    "df.rename({'level_0': 'title', 'level_1': 'category'}, axis='columns', inplace=True)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>category</th>\n",
       "      <th>variable</th>\n",
       "      <th>value</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Data Engineer</td>\n",
       "      <td>Big Data Technologies</td>\n",
       "      <td>Python</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Data Engineer</td>\n",
       "      <td>Cloud Computing Platforms</td>\n",
       "      <td>Python</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Data Engineer</td>\n",
       "      <td>Deep Learning Frameworks</td>\n",
       "      <td>Python</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Data Engineer</td>\n",
       "      <td>Machine Learning Application</td>\n",
       "      <td>Python</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Data Engineer</td>\n",
       "      <td>Other</td>\n",
       "      <td>Python</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           title                      category variable  value\n",
       "0  Data Engineer         Big Data Technologies   Python    NaN\n",
       "1  Data Engineer     Cloud Computing Platforms   Python    NaN\n",
       "2  Data Engineer      Deep Learning Frameworks   Python    NaN\n",
       "3  Data Engineer  Machine Learning Application   Python    NaN\n",
       "4  Data Engineer                         Other   Python    NaN"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "value_vars = df.columns.tolist()[2:] # the list of column names except the first two\n",
    "# Transform from wide to long for plotting\n",
    "df = pd.melt(df, id_vars=['title', 'category'], value_vars=value_vars)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>category</th>\n",
       "      <th>skill</th>\n",
       "      <th>frequency</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Data Engineer</td>\n",
       "      <td>Big Data Technologies</td>\n",
       "      <td>Python</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Data Engineer</td>\n",
       "      <td>Cloud Computing Platforms</td>\n",
       "      <td>Python</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Data Engineer</td>\n",
       "      <td>Deep Learning Frameworks</td>\n",
       "      <td>Python</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Data Engineer</td>\n",
       "      <td>Machine Learning Application</td>\n",
       "      <td>Python</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Data Engineer</td>\n",
       "      <td>Other</td>\n",
       "      <td>Python</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           title                      category   skill  frequency\n",
       "0  Data Engineer         Big Data Technologies  Python        NaN\n",
       "1  Data Engineer     Cloud Computing Platforms  Python        NaN\n",
       "2  Data Engineer      Deep Learning Frameworks  Python        NaN\n",
       "3  Data Engineer  Machine Learning Application  Python        NaN\n",
       "4  Data Engineer                         Other  Python        NaN"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Rename the last two columns\n",
    "df.rename({'variable': 'skill', 'value': 'frequency'}, axis='columns', inplace=True)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>category</th>\n",
       "      <th>skill</th>\n",
       "      <th>frequency</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Data Engineer</td>\n",
       "      <td>Programming Languages</td>\n",
       "      <td>Python</td>\n",
       "      <td>52.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Data Scientist</td>\n",
       "      <td>Programming Languages</td>\n",
       "      <td>Python</td>\n",
       "      <td>103.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>Machine Learning Engineer</td>\n",
       "      <td>Programming Languages</td>\n",
       "      <td>Python</td>\n",
       "      <td>71.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>Data Engineer</td>\n",
       "      <td>Programming Languages</td>\n",
       "      <td>R</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>Data Scientist</td>\n",
       "      <td>Programming Languages</td>\n",
       "      <td>R</td>\n",
       "      <td>19.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        title               category   skill  frequency\n",
       "5               Data Engineer  Programming Languages  Python       52.0\n",
       "12             Data Scientist  Programming Languages  Python      103.0\n",
       "19  Machine Learning Engineer  Programming Languages  Python       71.0\n",
       "26              Data Engineer  Programming Languages       R        5.0\n",
       "33             Data Scientist  Programming Languages       R       19.0"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Subset to non null values in the freq column\n",
    "df = df[df['frequency'].notnull()]\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>category</th>\n",
       "      <th>skill</th>\n",
       "      <th>frequency</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Data Engineer</td>\n",
       "      <td>Programming Languages</td>\n",
       "      <td>Python</td>\n",
       "      <td>52.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Data Scientist</td>\n",
       "      <td>Programming Languages</td>\n",
       "      <td>Python</td>\n",
       "      <td>103.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Machine Learning Engineer</td>\n",
       "      <td>Programming Languages</td>\n",
       "      <td>Python</td>\n",
       "      <td>71.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Data Engineer</td>\n",
       "      <td>Programming Languages</td>\n",
       "      <td>R</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Data Scientist</td>\n",
       "      <td>Programming Languages</td>\n",
       "      <td>R</td>\n",
       "      <td>19.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       title               category   skill  frequency\n",
       "0              Data Engineer  Programming Languages  Python       52.0\n",
       "1             Data Scientist  Programming Languages  Python      103.0\n",
       "2  Machine Learning Engineer  Programming Languages  Python       71.0\n",
       "3              Data Engineer  Programming Languages       R        5.0\n",
       "4             Data Scientist  Programming Languages       R       19.0"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Reset the index\n",
    "df.reset_index(drop=True, inplace=True)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "title        object\n",
       "category     object\n",
       "skill        object\n",
       "frequency     int32\n",
       "dtype: object"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df.astype({'frequency': int})\n",
    "df.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('skill_frequencies.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}