merge_sentiment_market_factor.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import pickle\n",
    "import copy\n",
    "from collections import Counter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "reload = False\n",
    "if reload:\n",
    "    sentiment_factors = pd.read_csv('./ALL.csv')\n",
    "    with open('sentiment_factors.pkl','wb') as f:\n",
    "        pickle.dump(sentiment_factors, f, pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('sentiment_factors.pkl', 'rb') as f:\n",
    "    sentiment_factors = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('market_factors_earning_enhanced.pkl', 'rb') as f:\n",
    "    market_factors = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(485498, 49)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentiment_factors.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4064834, 52)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "market_factors.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "market_factors['datetime'] = market_factors['time'].apply(lambda x:x[:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['datetime', 'assetCode', 'month', 'week', 'alert', 'article',\n",
       "       'alert_relevance_weighted_sentiment',\n",
       "       'alert_relevance_weighted_negative_sentiment',\n",
       "       'alert_relevance_weighted_positive_sentiment',\n",
       "       'alert_relevance_weighted_neutral_sentiment',\n",
       "       'alert_news_volume_sum_12H', 'alert_news_novelty_sum_12H',\n",
       "       'alert_news_volume_sum_24H', 'alert_news_novelty_sum_24H',\n",
       "       'alert_news_volume_sum_3D', 'alert_news_novelty_sum_3D',\n",
       "       'alert_news_volume_sum_5D', 'alert_news_novelty_sum_5D',\n",
       "       'alert_news_volume_sum_7D', 'alert_news_novelty_sum_7D',\n",
       "       'article_relevance_weighted_sentiment',\n",
       "       'article_relevance_weighted_negative_sentiment',\n",
       "       'article_relevance_weighted_positive_sentiment',\n",
       "       'article_relevance_weighted_neutral_sentiment',\n",
       "       'article_news_volume_sum_12H', 'article_news_novelty_sum_12H',\n",
       "       'article_news_volume_sum_24H', 'article_news_novelty_sum_24H',\n",
       "       'article_news_volume_sum_3D', 'article_news_novelty_sum_3D',\n",
       "       'article_news_volume_sum_5D', 'article_news_novelty_sum_5D',\n",
       "       'article_news_volume_sum_7D', 'article_news_novelty_sum_7D',\n",
       "       'relevance_weighted_sentiment', 'relevance_weighted_negative_sentiment',\n",
       "       'relevance_weighted_positive_sentiment',\n",
       "       'relevance_weighted_neutral_sentiment', 'news_volume_sum_12H',\n",
       "       'news_novelty_sum_12H', 'news_volume_sum_24H', 'news_novelty_sum_24H',\n",
       "       'news_volume_sum_3D', 'news_novelty_sum_3D', 'news_volume_sum_5D',\n",
       "       'news_novelty_sum_5D', 'news_volume_sum_7D', 'news_novelty_sum_7D',\n",
       "       'returnsOpenNextMktres10'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentiment_factors.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['index', 'Unnamed: 0', 'time', 'assetCode', 'assetName', 'volume',\n",
       "       'close', 'open', 'returnsClosePrevRaw1', 'returnsOpenPrevRaw1',\n",
       "       'returnsClosePrevMktres1', 'returnsOpenPrevMktres1',\n",
       "       'returnsClosePrevRaw10', 'returnsOpenPrevRaw10',\n",
       "       'returnsClosePrevMktres10', 'returnsOpenPrevMktres10',\n",
       "       'returnsOpenNextMktres10', 'universe', 'returnsVolumePrev1',\n",
       "       'returnsVolumePrev10', 'CloseMA5', 'CloseMA10', 'CloseMA20',\n",
       "       'returnsCloseMA5', 'returnsCloseMA10', 'returnsCloseMA20', 'VolumeMA5',\n",
       "       'VolumeMA10', 'VolumeMA20', 'returnsVolumeMA5', 'returnsVolumeMA10',\n",
       "       'returnsVolumeMA20', 'date', 'earning_period_flag', 'reversal',\n",
       "       'modified_earning_flag', 'reverse_returnsClosePrevRaw1',\n",
       "       'reverse_returnsOpenPrevRaw1', 'reverse_returnsClosePrevMktres1',\n",
       "       'reverse_returnsOpenPrevMktres1', 'reverse_returnsCloseMA5',\n",
       "       'reverse_VolumeMA5', 'reverse_VolumeMA10', 'reverse_VolumeMA20',\n",
       "       'earnings_enhanced_reverse_returnsClosePrevRaw1',\n",
       "       'earnings_enhanced_reverse_returnsOpenPrevRaw1',\n",
       "       'earnings_enhanced_reverse_returnsClosePrevMktres1',\n",
       "       'earnings_enhanced_reverse_returnsOpenPrevMktres1',\n",
       "       'earnings_enhanced_reverse_returnsCloseMA5',\n",
       "       'earnings_enhanced_reverse_VolumeMA5',\n",
       "       'earnings_enhanced_reverse_VolumeMA10',\n",
       "       'earnings_enhanced_reverse_VolumeMA20', 'datetime'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "market_factors.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>datetime</th>\n",
       "      <th>assetCode</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2007-02-01</td>\n",
       "      <td>A.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2007-02-01</td>\n",
       "      <td>MRO.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2007-02-01</td>\n",
       "      <td>MS.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2007-02-01</td>\n",
       "      <td>MSFT.O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2007-02-01</td>\n",
       "      <td>MYL.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>2007-02-01</td>\n",
       "      <td>NDAQ.O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>2007-02-01</td>\n",
       "      <td>NKE.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>2007-02-01</td>\n",
       "      <td>NMX.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>2007-02-01</td>\n",
       "      <td>NOC.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>2007-02-01</td>\n",
       "      <td>MO.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>2007-02-01</td>\n",
       "      <td>NOK.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>2007-02-01</td>\n",
       "      <td>NVS.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>2007-02-01</td>\n",
       "      <td>NYT.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>2007-02-01</td>\n",
       "      <td>OMC.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>2007-02-01</td>\n",
       "      <td>PD.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>2007-02-01</td>\n",
       "      <td>PFE.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>2007-02-01</td>\n",
       "      <td>PFG.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>2007-02-01</td>\n",
       "      <td>PG.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>2007-02-01</td>\n",
       "      <td>PKI.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>2007-02-01</td>\n",
       "      <td>NVDA.O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>2007-02-01</td>\n",
       "      <td>PKX.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>2007-02-01</td>\n",
       "      <td>MMC.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>2007-02-01</td>\n",
       "      <td>MCO.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>2007-02-01</td>\n",
       "      <td>HSY.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>2007-02-01</td>\n",
       "      <td>IBM.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>2007-02-01</td>\n",
       "      <td>JBLU.O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>2007-02-01</td>\n",
       "      <td>JCP.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>2007-02-01</td>\n",
       "      <td>JEC.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>2007-02-01</td>\n",
       "      <td>JNC.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>2007-02-01</td>\n",
       "      <td>JNJ.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>485468</th>\n",
       "      <td>2016-12-30</td>\n",
       "      <td>ABX.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>485469</th>\n",
       "      <td>2016-12-30</td>\n",
       "      <td>ADI.O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>485470</th>\n",
       "      <td>2016-12-30</td>\n",
       "      <td>ADM.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>485471</th>\n",
       "      <td>2016-12-30</td>\n",
       "      <td>AGN.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>485472</th>\n",
       "      <td>2016-12-30</td>\n",
       "      <td>AMAT.O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>485473</th>\n",
       "      <td>2016-12-30</td>\n",
       "      <td>AMZN.O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>485474</th>\n",
       "      <td>2016-12-30</td>\n",
       "      <td>ASML.O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>485475</th>\n",
       "      <td>2016-12-30</td>\n",
       "      <td>BABA.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>485476</th>\n",
       "      <td>2016-12-30</td>\n",
       "      <td>BBL.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>485477</th>\n",
       "      <td>2016-12-30</td>\n",
       "      <td>BCS.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>485478</th>\n",
       "      <td>2016-12-30</td>\n",
       "      <td>BG.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>485479</th>\n",
       "      <td>2016-12-30</td>\n",
       "      <td>BHI.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>485480</th>\n",
       "      <td>2016-12-30</td>\n",
       "      <td>GS.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>485481</th>\n",
       "      <td>2016-12-30</td>\n",
       "      <td>BHP.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>485482</th>\n",
       "      <td>2016-12-30</td>\n",
       "      <td>CNI.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>485483</th>\n",
       "      <td>2016-12-30</td>\n",
       "      <td>CRUS.O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>485484</th>\n",
       "      <td>2016-12-30</td>\n",
       "      <td>CS.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>485485</th>\n",
       "      <td>2016-12-30</td>\n",
       "      <td>CVX.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>485486</th>\n",
       "      <td>2016-12-30</td>\n",
       "      <td>DAL.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>485487</th>\n",
       "      <td>2016-12-30</td>\n",
       "      <td>DB.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>485488</th>\n",
       "      <td>2016-12-30</td>\n",
       "      <td>DIS.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>485489</th>\n",
       "      <td>2016-12-30</td>\n",
       "      <td>DISH.O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>485490</th>\n",
       "      <td>2016-12-30</td>\n",
       "      <td>ENB.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>485491</th>\n",
       "      <td>2016-12-30</td>\n",
       "      <td>ENDP.O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>485492</th>\n",
       "      <td>2016-12-30</td>\n",
       "      <td>ESRX.O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>485493</th>\n",
       "      <td>2016-12-30</td>\n",
       "      <td>FB.O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>485494</th>\n",
       "      <td>2016-12-30</td>\n",
       "      <td>FCX.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>485495</th>\n",
       "      <td>2016-12-30</td>\n",
       "      <td>FOXA.O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>485496</th>\n",
       "      <td>2016-12-30</td>\n",
       "      <td>C.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>485497</th>\n",
       "      <td>2016-12-30</td>\n",
       "      <td>WPZ.N</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>485498 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          datetime assetCode\n",
       "0       2007-02-01       A.N\n",
       "1       2007-02-01     MRO.N\n",
       "2       2007-02-01      MS.N\n",
       "3       2007-02-01    MSFT.O\n",
       "4       2007-02-01     MYL.N\n",
       "5       2007-02-01    NDAQ.O\n",
       "6       2007-02-01     NKE.N\n",
       "7       2007-02-01     NMX.N\n",
       "8       2007-02-01     NOC.N\n",
       "9       2007-02-01      MO.N\n",
       "10      2007-02-01     NOK.N\n",
       "11      2007-02-01     NVS.N\n",
       "12      2007-02-01     NYT.N\n",
       "13      2007-02-01     OMC.N\n",
       "14      2007-02-01      PD.N\n",
       "15      2007-02-01     PFE.N\n",
       "16      2007-02-01     PFG.N\n",
       "17      2007-02-01      PG.N\n",
       "18      2007-02-01     PKI.N\n",
       "19      2007-02-01    NVDA.O\n",
       "20      2007-02-01     PKX.N\n",
       "21      2007-02-01     MMC.N\n",
       "22      2007-02-01     MCO.N\n",
       "23      2007-02-01     HSY.N\n",
       "24      2007-02-01     IBM.N\n",
       "25      2007-02-01    JBLU.O\n",
       "26      2007-02-01     JCP.N\n",
       "27      2007-02-01     JEC.N\n",
       "28      2007-02-01     JNC.N\n",
       "29      2007-02-01     JNJ.N\n",
       "...            ...       ...\n",
       "485468  2016-12-30     ABX.N\n",
       "485469  2016-12-30     ADI.O\n",
       "485470  2016-12-30     ADM.N\n",
       "485471  2016-12-30     AGN.N\n",
       "485472  2016-12-30    AMAT.O\n",
       "485473  2016-12-30    AMZN.O\n",
       "485474  2016-12-30    ASML.O\n",
       "485475  2016-12-30    BABA.N\n",
       "485476  2016-12-30     BBL.N\n",
       "485477  2016-12-30     BCS.N\n",
       "485478  2016-12-30      BG.N\n",
       "485479  2016-12-30     BHI.N\n",
       "485480  2016-12-30      GS.N\n",
       "485481  2016-12-30     BHP.N\n",
       "485482  2016-12-30     CNI.N\n",
       "485483  2016-12-30    CRUS.O\n",
       "485484  2016-12-30      CS.N\n",
       "485485  2016-12-30     CVX.N\n",
       "485486  2016-12-30     DAL.N\n",
       "485487  2016-12-30      DB.N\n",
       "485488  2016-12-30     DIS.N\n",
       "485489  2016-12-30    DISH.O\n",
       "485490  2016-12-30     ENB.N\n",
       "485491  2016-12-30    ENDP.O\n",
       "485492  2016-12-30    ESRX.O\n",
       "485493  2016-12-30      FB.O\n",
       "485494  2016-12-30     FCX.N\n",
       "485495  2016-12-30    FOXA.O\n",
       "485496  2016-12-30       C.N\n",
       "485497  2016-12-30     WPZ.N\n",
       "\n",
       "[485498 rows x 2 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentiment_factors[['datetime', 'assetCode']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>datetime</th>\n",
       "      <th>assetCode</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2007-02-01</td>\n",
       "      <td>A.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2007-02-01</td>\n",
       "      <td>AAI.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2007-02-01</td>\n",
       "      <td>AAP.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2007-02-01</td>\n",
       "      <td>AAPL.O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2007-02-01</td>\n",
       "      <td>ABB.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>2007-02-01</td>\n",
       "      <td>ABC.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>2007-02-01</td>\n",
       "      <td>ABD.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>2007-02-01</td>\n",
       "      <td>ABM.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>2007-02-01</td>\n",
       "      <td>ABT.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>2007-02-01</td>\n",
       "      <td>ABV.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>2007-02-01</td>\n",
       "      <td>ABY.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>2007-02-01</td>\n",
       "      <td>ACF.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>2007-02-01</td>\n",
       "      <td>ACH.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>2007-02-01</td>\n",
       "      <td>ACL.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>2007-02-01</td>\n",
       "      <td>ACS.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>2007-02-01</td>\n",
       "      <td>ACTS.O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>2007-02-01</td>\n",
       "      <td>ACXM.O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>2007-02-01</td>\n",
       "      <td>AD.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>2007-02-01</td>\n",
       "      <td>ADBE.O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>2007-02-01</td>\n",
       "      <td>ADCT.O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>2007-02-01</td>\n",
       "      <td>ADI.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>2007-02-01</td>\n",
       "      <td>ADM.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>2007-02-01</td>\n",
       "      <td>ADP.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>2007-02-01</td>\n",
       "      <td>ADS.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>2007-02-01</td>\n",
       "      <td>ADSK.O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>2007-02-01</td>\n",
       "      <td>ADTN.O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>2007-02-01</td>\n",
       "      <td>AEA.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>2007-02-01</td>\n",
       "      <td>AEE.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>2007-02-01</td>\n",
       "      <td>AEG.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>2007-02-01</td>\n",
       "      <td>AEIS.O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4064804</th>\n",
       "      <td>2016-12-30</td>\n",
       "      <td>XLNX.O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4064805</th>\n",
       "      <td>2016-12-30</td>\n",
       "      <td>XLRN.O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4064806</th>\n",
       "      <td>2016-12-30</td>\n",
       "      <td>XNCR.O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4064807</th>\n",
       "      <td>2016-12-30</td>\n",
       "      <td>XOG.O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4064808</th>\n",
       "      <td>2016-12-30</td>\n",
       "      <td>XOM.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4064809</th>\n",
       "      <td>2016-12-30</td>\n",
       "      <td>XON.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4064810</th>\n",
       "      <td>2016-12-30</td>\n",
       "      <td>XPO.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4064811</th>\n",
       "      <td>2016-12-30</td>\n",
       "      <td>XRAY.O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4064812</th>\n",
       "      <td>2016-12-30</td>\n",
       "      <td>XXIA.O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4064813</th>\n",
       "      <td>2016-12-30</td>\n",
       "      <td>XYL.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4064814</th>\n",
       "      <td>2016-12-30</td>\n",
       "      <td>YELP.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4064815</th>\n",
       "      <td>2016-12-30</td>\n",
       "      <td>YHOO.O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4064816</th>\n",
       "      <td>2016-12-30</td>\n",
       "      <td>YNDX.O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4064817</th>\n",
       "      <td>2016-12-30</td>\n",
       "      <td>YPF.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4064818</th>\n",
       "      <td>2016-12-30</td>\n",
       "      <td>YRD.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4064819</th>\n",
       "      <td>2016-12-30</td>\n",
       "      <td>YUM.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4064820</th>\n",
       "      <td>2016-12-30</td>\n",
       "      <td>YUMC.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4064821</th>\n",
       "      <td>2016-12-30</td>\n",
       "      <td>YY.O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4064822</th>\n",
       "      <td>2016-12-30</td>\n",
       "      <td>Z.O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4064823</th>\n",
       "      <td>2016-12-30</td>\n",
       "      <td>ZAYO.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4064824</th>\n",
       "      <td>2016-12-30</td>\n",
       "      <td>ZBH.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4064825</th>\n",
       "      <td>2016-12-30</td>\n",
       "      <td>ZBRA.O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4064826</th>\n",
       "      <td>2016-12-30</td>\n",
       "      <td>ZEN.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4064827</th>\n",
       "      <td>2016-12-30</td>\n",
       "      <td>ZG.O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4064828</th>\n",
       "      <td>2016-12-30</td>\n",
       "      <td>ZION.O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4064829</th>\n",
       "      <td>2016-12-30</td>\n",
       "      <td>ZIOP.O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4064830</th>\n",
       "      <td>2016-12-30</td>\n",
       "      <td>ZLTQ.O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4064831</th>\n",
       "      <td>2016-12-30</td>\n",
       "      <td>ZNGA.O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4064832</th>\n",
       "      <td>2016-12-30</td>\n",
       "      <td>ZTO.N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4064833</th>\n",
       "      <td>2016-12-30</td>\n",
       "      <td>ZTS.N</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4064834 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           datetime assetCode\n",
       "0        2007-02-01       A.N\n",
       "1        2007-02-01     AAI.N\n",
       "2        2007-02-01     AAP.N\n",
       "3        2007-02-01    AAPL.O\n",
       "4        2007-02-01     ABB.N\n",
       "5        2007-02-01     ABC.N\n",
       "6        2007-02-01     ABD.N\n",
       "7        2007-02-01     ABM.N\n",
       "8        2007-02-01     ABT.N\n",
       "9        2007-02-01     ABV.N\n",
       "10       2007-02-01     ABY.N\n",
       "11       2007-02-01     ACF.N\n",
       "12       2007-02-01     ACH.N\n",
       "13       2007-02-01     ACL.N\n",
       "14       2007-02-01     ACS.N\n",
       "15       2007-02-01    ACTS.O\n",
       "16       2007-02-01    ACXM.O\n",
       "17       2007-02-01      AD.N\n",
       "18       2007-02-01    ADBE.O\n",
       "19       2007-02-01    ADCT.O\n",
       "20       2007-02-01     ADI.N\n",
       "21       2007-02-01     ADM.N\n",
       "22       2007-02-01     ADP.N\n",
       "23       2007-02-01     ADS.N\n",
       "24       2007-02-01    ADSK.O\n",
       "25       2007-02-01    ADTN.O\n",
       "26       2007-02-01     AEA.N\n",
       "27       2007-02-01     AEE.N\n",
       "28       2007-02-01     AEG.N\n",
       "29       2007-02-01    AEIS.O\n",
       "...             ...       ...\n",
       "4064804  2016-12-30    XLNX.O\n",
       "4064805  2016-12-30    XLRN.O\n",
       "4064806  2016-12-30    XNCR.O\n",
       "4064807  2016-12-30     XOG.O\n",
       "4064808  2016-12-30     XOM.N\n",
       "4064809  2016-12-30     XON.N\n",
       "4064810  2016-12-30     XPO.N\n",
       "4064811  2016-12-30    XRAY.O\n",
       "4064812  2016-12-30    XXIA.O\n",
       "4064813  2016-12-30     XYL.N\n",
       "4064814  2016-12-30    YELP.N\n",
       "4064815  2016-12-30    YHOO.O\n",
       "4064816  2016-12-30    YNDX.O\n",
       "4064817  2016-12-30     YPF.N\n",
       "4064818  2016-12-30     YRD.N\n",
       "4064819  2016-12-30     YUM.N\n",
       "4064820  2016-12-30    YUMC.N\n",
       "4064821  2016-12-30      YY.O\n",
       "4064822  2016-12-30       Z.O\n",
       "4064823  2016-12-30    ZAYO.N\n",
       "4064824  2016-12-30     ZBH.N\n",
       "4064825  2016-12-30    ZBRA.O\n",
       "4064826  2016-12-30     ZEN.N\n",
       "4064827  2016-12-30      ZG.O\n",
       "4064828  2016-12-30    ZION.O\n",
       "4064829  2016-12-30    ZIOP.O\n",
       "4064830  2016-12-30    ZLTQ.O\n",
       "4064831  2016-12-30    ZNGA.O\n",
       "4064832  2016-12-30     ZTO.N\n",
       "4064833  2016-12-30     ZTS.N\n",
       "\n",
       "[4064834 rows x 2 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "market_factors[[ 'datetime', 'assetCode']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "datetime    assetCode\n",
       "2007-02-01  A.N          0.000000\n",
       "            ABB.N        0.000000\n",
       "            ABT.N        1.000000\n",
       "            ABV.N        1.000000\n",
       "            ACH.N        1.000000\n",
       "            ADBE.O       0.000000\n",
       "            ADCT.O      -0.286423\n",
       "            ADM.N        0.691113\n",
       "            AFFX.O       0.000000\n",
       "            AG.N         1.000000\n",
       "            AGN.N       -1.000000\n",
       "            ALK.N        1.000000\n",
       "            AMZN.O      -0.634273\n",
       "            APOL.O       0.000000\n",
       "            ASN.N        0.000000\n",
       "            ATI.N        0.000000\n",
       "            AXP.N        1.000000\n",
       "            AZN.N        0.543401\n",
       "            BA.N        -0.462828\n",
       "            BAC.N       -0.454164\n",
       "            BBY.N        0.540663\n",
       "            BCR.N        1.000000\n",
       "            BCS.N       -0.022413\n",
       "            BDX.N        0.000000\n",
       "            BEC.N        0.000000\n",
       "            BHP.N        0.309193\n",
       "            BIDU.O      -1.000000\n",
       "            BK.N        -1.000000\n",
       "            BMC.N        1.000000\n",
       "            BMY.N       -0.569369\n",
       "                           ...   \n",
       "2016-12-30  JNJ.N        0.225233\n",
       "            JPM.N       -0.720825\n",
       "            LAZ.N       -1.000000\n",
       "            MCK.N       -1.000000\n",
       "            MFC.N        1.000000\n",
       "            MON.N        1.000000\n",
       "            MS.N         1.000000\n",
       "            MSFT.O      -0.196157\n",
       "            MU.O        -0.866323\n",
       "            MYL.O        0.391459\n",
       "            NEM.N        0.000000\n",
       "            NFLX.O       1.000000\n",
       "            NVDA.O      -0.944834\n",
       "            OKE.N       -0.944834\n",
       "            PDCE.O      -1.000000\n",
       "            PFE.N       -0.269677\n",
       "            QCOM.O      -0.603757\n",
       "            QRVO.O      -1.000000\n",
       "            RBS.N       -1.000000\n",
       "            RDSb.N       1.000000\n",
       "            RIO.N        1.000000\n",
       "            SGEN.O      -1.000000\n",
       "            SWKS.O      -0.866323\n",
       "            T.N          1.000000\n",
       "            TD.N         1.000000\n",
       "            TEF.N        1.000000\n",
       "            TSM.N        1.000000\n",
       "            UBS.N        1.000000\n",
       "            UNH.N        1.000000\n",
       "            WPZ.N       -0.248780\n",
       "Name: alert_relevance_weighted_sentiment, Length: 485498, dtype: float64"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentiment_factors.groupby(['datetime','assetCode'])['alert_relevance_weighted_sentiment'].sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "merged_factors = pd.merge(market_factors,sentiment_factors,how = 'left',left_on = ['datetime','assetCode'],right_on = ['datetime','assetCode'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['index', 'Unnamed: 0', 'time', 'assetCode', 'assetName', 'volume',\n",
       "       'close', 'open', 'returnsClosePrevRaw1', 'returnsOpenPrevRaw1',\n",
       "       'returnsClosePrevMktres1', 'returnsOpenPrevMktres1',\n",
       "       'returnsClosePrevRaw10', 'returnsOpenPrevRaw10',\n",
       "       'returnsClosePrevMktres10', 'returnsOpenPrevMktres10',\n",
       "       'returnsOpenNextMktres10_x', 'universe', 'returnsVolumePrev1',\n",
       "       'returnsVolumePrev10', 'CloseMA5', 'CloseMA10', 'CloseMA20',\n",
       "       'returnsCloseMA5', 'returnsCloseMA10', 'returnsCloseMA20', 'VolumeMA5',\n",
       "       'VolumeMA10', 'VolumeMA20', 'returnsVolumeMA5', 'returnsVolumeMA10',\n",
       "       'returnsVolumeMA20', 'date', 'earning_period_flag', 'reversal',\n",
       "       'modified_earning_flag', 'reverse_returnsClosePrevRaw1',\n",
       "       'reverse_returnsOpenPrevRaw1', 'reverse_returnsClosePrevMktres1',\n",
       "       'reverse_returnsOpenPrevMktres1', 'reverse_returnsCloseMA5',\n",
       "       'reverse_VolumeMA5', 'reverse_VolumeMA10', 'reverse_VolumeMA20',\n",
       "       'earnings_enhanced_reverse_returnsClosePrevRaw1',\n",
       "       'earnings_enhanced_reverse_returnsOpenPrevRaw1',\n",
       "       'earnings_enhanced_reverse_returnsClosePrevMktres1',\n",
       "       'earnings_enhanced_reverse_returnsOpenPrevMktres1',\n",
       "       'earnings_enhanced_reverse_returnsCloseMA5',\n",
       "       'earnings_enhanced_reverse_VolumeMA5',\n",
       "       'earnings_enhanced_reverse_VolumeMA10',\n",
       "       'earnings_enhanced_reverse_VolumeMA20', 'datetime', 'month', 'week',\n",
       "       'alert', 'article', 'alert_relevance_weighted_sentiment',\n",
       "       'alert_relevance_weighted_negative_sentiment',\n",
       "       'alert_relevance_weighted_positive_sentiment',\n",
       "       'alert_relevance_weighted_neutral_sentiment',\n",
       "       'alert_news_volume_sum_12H', 'alert_news_novelty_sum_12H',\n",
       "       'alert_news_volume_sum_24H', 'alert_news_novelty_sum_24H',\n",
       "       'alert_news_volume_sum_3D', 'alert_news_novelty_sum_3D',\n",
       "       'alert_news_volume_sum_5D', 'alert_news_novelty_sum_5D',\n",
       "       'alert_news_volume_sum_7D', 'alert_news_novelty_sum_7D',\n",
       "       'article_relevance_weighted_sentiment',\n",
       "       'article_relevance_weighted_negative_sentiment',\n",
       "       'article_relevance_weighted_positive_sentiment',\n",
       "       'article_relevance_weighted_neutral_sentiment',\n",
       "       'article_news_volume_sum_12H', 'article_news_novelty_sum_12H',\n",
       "       'article_news_volume_sum_24H', 'article_news_novelty_sum_24H',\n",
       "       'article_news_volume_sum_3D', 'article_news_novelty_sum_3D',\n",
       "       'article_news_volume_sum_5D', 'article_news_novelty_sum_5D',\n",
       "       'article_news_volume_sum_7D', 'article_news_novelty_sum_7D',\n",
       "       'relevance_weighted_sentiment', 'relevance_weighted_negative_sentiment',\n",
       "       'relevance_weighted_positive_sentiment',\n",
       "       'relevance_weighted_neutral_sentiment', 'news_volume_sum_12H',\n",
       "       'news_novelty_sum_12H', 'news_volume_sum_24H', 'news_novelty_sum_24H',\n",
       "       'news_volume_sum_3D', 'news_novelty_sum_3D', 'news_volume_sum_5D',\n",
       "       'news_novelty_sum_5D', 'news_volume_sum_7D', 'news_novelty_sum_7D',\n",
       "       'returnsOpenNextMktres10_y'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merged_factors.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "merged_factors['alert'].fillna(0,inplace=True)\n",
    "merged_factors['article'].fillna(0,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "merged_factors['news_coverage_flag'] = merged_factors['alert']+merged_factors['article']*3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0           6.0\n",
       "1           0.0\n",
       "2           0.0\n",
       "3           0.0\n",
       "4           3.0\n",
       "5           0.0\n",
       "6           0.0\n",
       "7           0.0\n",
       "8           6.0\n",
       "9           9.0\n",
       "10          0.0\n",
       "11          0.0\n",
       "12         14.0\n",
       "13          0.0\n",
       "14          0.0\n",
       "15          0.0\n",
       "16          0.0\n",
       "17          0.0\n",
       "18          3.0\n",
       "19         12.0\n",
       "20          0.0\n",
       "21         46.0\n",
       "22          0.0\n",
       "23          0.0\n",
       "24          0.0\n",
       "25          0.0\n",
       "26          0.0\n",
       "27          0.0\n",
       "28          0.0\n",
       "29          0.0\n",
       "           ... \n",
       "4064804     0.0\n",
       "4064805     0.0\n",
       "4064806     0.0\n",
       "4064807     0.0\n",
       "4064808     0.0\n",
       "4064809     0.0\n",
       "4064810     0.0\n",
       "4064811     0.0\n",
       "4064812     0.0\n",
       "4064813     0.0\n",
       "4064814     0.0\n",
       "4064815     0.0\n",
       "4064816     0.0\n",
       "4064817     0.0\n",
       "4064818     0.0\n",
       "4064819     0.0\n",
       "4064820     0.0\n",
       "4064821     0.0\n",
       "4064822     0.0\n",
       "4064823     0.0\n",
       "4064824     0.0\n",
       "4064825     0.0\n",
       "4064826     0.0\n",
       "4064827     0.0\n",
       "4064828     0.0\n",
       "4064829     0.0\n",
       "4064830     0.0\n",
       "4064831     0.0\n",
       "4064832     0.0\n",
       "4064833     0.0\n",
       "Name: news_coverage_flag, Length: 4064834, dtype: float64"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merged_factors['news_coverage_flag']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "merged_factors_fillna = copy.deepcopy(merged_factors)\n",
    "for signal in list(sentiment_factors.columns)[6:-1]:\n",
    "    merged_factors_fillna[signal].fillna(merged_factors_fillna[signal].median())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('merged_factors.pkl','wb') as f:\n",
    "    pickle.dump(merged_factors, f, pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('merged_fillna.pkl','wb') as f:\n",
    "    pickle.dump(merged_factors_fillna, f, pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "101"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(merged_factors_fillna.columns)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}