Data_init3.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Filter FAC, Electro jets, Density normalization, Half orbit means and pyamps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import pickle\n",
    "import My_functions_v2 as my"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "DNS = pd.read_pickle('/home/simon/Desktop/Bachelor_project/data/pkl_files/DNS.pkl')\n",
    "FAC = pd.read_pickle('/home/simon/Desktop/Bachelor_project/data/pkl_files/FAC_dual.pkl')\n",
    "omni = pd.read_pickle('/home/simon/Desktop/Bachelor_project/data/pkl_files/omni.pkl')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Filter FAC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1542 observation out off 3802595 are removed\n"
     ]
    }
   ],
   "source": [
    "FAC_filter= my.filter_FAC(FAC,dt = 10 ,Flags_B=10002,Flags_q=440404)\n",
    "FAC_filter.to_pickle('/home/simon/Desktop/Bachelor_project/data/pkl_files/FAC_filter.pkl')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Electrojets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "Electro_jets = my.get_jets(FAC_filter) \n",
    "solar_time = DNS.Local_solar_time.reindex(Electro_jets.index,method='nearest')\n",
    "Electro_jets = pd.concat([Electro_jets,solar_time], axis=1)\n",
    "Electro_jets.to_pickle('/home/simon/Desktop/Bachelor_project/data/pkl_files/Electro_jets.pkl')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Density normalization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "normFactors = np.genfromtxt('/home/simon/Desktop/Bachelor_project/data/normalization_factors.csv', delimiter=',')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "DNS_norm = DNS.copy()\n",
    "Adjusted_density = DNS.Density*normFactors[:,0]*normFactors[:,1]\n",
    "DNS_norm.loc[:,'Density'] = Adjusted_density.values\n",
    "DNS_norm.to_pickle('/home/simon/Desktop/Bachelor_project/data/pkl_files/DNS_normalized.pkl')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Half orbit means "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "FAC_mean=my.orbit_means(FAC_filter)\n",
    "FAC_power = my.orbit_means(FAC_filter,mode='power')\n",
    "DNS_mean=my.orbit_means(DNS)\n",
    "DNS_norm_mean = my.orbit_means(DNS_norm)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Remove incomplete orbits by considering the time "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "FAC: 54 orbits out of 1732 removed\n",
      "FAC_power: 54 orbits out of 1732 removed\n",
      "DNS: 24 orbits out of 1732 removed\n",
      "DNS_norm: 24 orbits out of 1732 removed\n"
     ]
    }
   ],
   "source": [
    "N_FAC = len(FAC_mean)\n",
    "N_power = len(FAC_power)\n",
    "N_DNS = len(DNS_mean)\n",
    "N_DNS_norm = len(DNS_norm_mean)\n",
    "\n",
    "FAC_mean=FAC_mean.loc[abs(FAC_mean.Delta_time-FAC_mean.Delta_time.median())<360]\n",
    "FAC_power=FAC_power.loc[abs(FAC_power.Delta_time-FAC_power.Delta_time.median())<360]\n",
    "DNS_mean=DNS_mean.loc[abs(DNS_mean.Delta_time-DNS_mean.Delta_time.median())<360]\n",
    "DNS_norm_mean=DNS_norm_mean.loc[abs(DNS_norm_mean.Delta_time-DNS_norm_mean.Delta_time.median())<360]\n",
    "\n",
    "print(\"FAC: %d orbits out of %d removed\" % (N_FAC-len(FAC_mean),N_FAC))\n",
    "print(\"FAC_power: %d orbits out of %d removed\" % (N_power-len(FAC_power),N_power))\n",
    "print(\"DNS: %d orbits out of %d removed\" % (N_DNS-len(DNS_mean),N_DNS))\n",
    "print(\"DNS_norm: %d orbits out of %d removed\" % (N_DNS_norm-len(DNS_norm_mean),N_DNS_norm))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Align measument with those of DNS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "df1 = DNS_norm_mean.Density.reindex(DNS_mean.index,method='ffill')\n",
    "df1.name = 'Density_norm'\n",
    "\n",
    "df2 = FAC_mean.FAC.reindex(DNS_mean.index,method='nearest')\n",
    "df2.name = 'FAC'\n",
    "\n",
    "df3 = FAC_power.FAC.reindex(DNS_mean.index,method='nearest')\n",
    "df3.name = 'FAC_power'\n",
    "\n",
    "Half_orbit = pd.concat([df1,df2,df3,DNS_mean.iloc[:,0:4]], axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "add omni data and save "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "omni_alligned = omni.reindex(Half_orbit.index,method='ffill')\n",
    "Half_orbit = pd.concat([Half_orbit,omni_alligned], axis=1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "my.add_pyamps_currents(Half_orbit)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "Half_orbit.to_pickle('/home/simon/Desktop/Bachelor_project/data/pkl_files/half_orbit_means.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Density_norm</th>\n",
       "      <th>FAC</th>\n",
       "      <th>FAC_power</th>\n",
       "      <th>Density</th>\n",
       "      <th>Altitude</th>\n",
       "      <th>Orbit_nr</th>\n",
       "      <th>Hemisphere</th>\n",
       "      <th>AP_INDEX</th>\n",
       "      <th>Kp</th>\n",
       "      <th>F10_INDEX</th>\n",
       "      <th>BY_GSM</th>\n",
       "      <th>BZ_GSM</th>\n",
       "      <th>Bulk_speed</th>\n",
       "      <th>DST</th>\n",
       "      <th>AE</th>\n",
       "      <th>F10_Mean</th>\n",
       "      <th>PyAmps</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2015-01-02 00:50:50</th>\n",
       "      <td>5.932167e-13</td>\n",
       "      <td>0.104478</td>\n",
       "      <td>0.081088</td>\n",
       "      <td>6.468488e-13</td>\n",
       "      <td>464222.287865</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>141.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-2.1</td>\n",
       "      <td>433.0</td>\n",
       "      <td>-22.0</td>\n",
       "      <td>175.0</td>\n",
       "      <td>129.029218</td>\n",
       "      <td>1.769016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-01-02 01:37:45</th>\n",
       "      <td>7.720531e-13</td>\n",
       "      <td>0.139248</td>\n",
       "      <td>0.138844</td>\n",
       "      <td>6.987142e-13</td>\n",
       "      <td>475492.964700</td>\n",
       "      <td>1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>141.0</td>\n",
       "      <td>-0.6</td>\n",
       "      <td>-1.7</td>\n",
       "      <td>433.0</td>\n",
       "      <td>-21.0</td>\n",
       "      <td>115.0</td>\n",
       "      <td>129.032511</td>\n",
       "      <td>2.962752</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-01-02 02:24:40</th>\n",
       "      <td>5.577984e-13</td>\n",
       "      <td>0.113231</td>\n",
       "      <td>0.152365</td>\n",
       "      <td>6.089699e-13</td>\n",
       "      <td>464224.157119</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>141.0</td>\n",
       "      <td>-2.2</td>\n",
       "      <td>-2.4</td>\n",
       "      <td>434.0</td>\n",
       "      <td>-16.0</td>\n",
       "      <td>155.0</td>\n",
       "      <td>129.035803</td>\n",
       "      <td>1.880936</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-01-02 03:11:35</th>\n",
       "      <td>7.597623e-13</td>\n",
       "      <td>0.181961</td>\n",
       "      <td>0.320141</td>\n",
       "      <td>6.858343e-13</td>\n",
       "      <td>475506.057772</td>\n",
       "      <td>2.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>141.0</td>\n",
       "      <td>-3.0</td>\n",
       "      <td>-2.3</td>\n",
       "      <td>433.0</td>\n",
       "      <td>-7.0</td>\n",
       "      <td>79.0</td>\n",
       "      <td>129.039095</td>\n",
       "      <td>4.111182</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-01-02 03:58:30</th>\n",
       "      <td>5.691719e-13</td>\n",
       "      <td>0.075313</td>\n",
       "      <td>0.027412</td>\n",
       "      <td>6.237317e-13</td>\n",
       "      <td>464035.397442</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>141.0</td>\n",
       "      <td>-3.0</td>\n",
       "      <td>-2.3</td>\n",
       "      <td>433.0</td>\n",
       "      <td>-7.0</td>\n",
       "      <td>79.0</td>\n",
       "      <td>129.039095</td>\n",
       "      <td>1.841245</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-01-02 04:45:25</th>\n",
       "      <td>7.214728e-13</td>\n",
       "      <td>0.084554</td>\n",
       "      <td>0.050610</td>\n",
       "      <td>6.492501e-13</td>\n",
       "      <td>475514.819681</td>\n",
       "      <td>3.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>141.0</td>\n",
       "      <td>-3.0</td>\n",
       "      <td>-1.7</td>\n",
       "      <td>429.0</td>\n",
       "      <td>-3.0</td>\n",
       "      <td>48.0</td>\n",
       "      <td>129.042387</td>\n",
       "      <td>3.673843</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-01-02 05:32:20</th>\n",
       "      <td>5.458171e-13</td>\n",
       "      <td>0.090864</td>\n",
       "      <td>0.060029</td>\n",
       "      <td>5.960482e-13</td>\n",
       "      <td>464242.140820</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>141.0</td>\n",
       "      <td>-1.8</td>\n",
       "      <td>-1.1</td>\n",
       "      <td>416.0</td>\n",
       "      <td>-4.0</td>\n",
       "      <td>49.0</td>\n",
       "      <td>129.045679</td>\n",
       "      <td>1.109322</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-01-02 06:19:15</th>\n",
       "      <td>7.206711e-13</td>\n",
       "      <td>0.085828</td>\n",
       "      <td>0.040027</td>\n",
       "      <td>6.465145e-13</td>\n",
       "      <td>475520.773958</td>\n",
       "      <td>4.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1.3</td>\n",
       "      <td>141.0</td>\n",
       "      <td>-2.6</td>\n",
       "      <td>-1.1</td>\n",
       "      <td>416.0</td>\n",
       "      <td>-4.0</td>\n",
       "      <td>73.0</td>\n",
       "      <td>129.048972</td>\n",
       "      <td>3.038630</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-01-02 07:06:10</th>\n",
       "      <td>5.683820e-13</td>\n",
       "      <td>0.048973</td>\n",
       "      <td>0.015818</td>\n",
       "      <td>6.189799e-13</td>\n",
       "      <td>464277.002808</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1.3</td>\n",
       "      <td>141.0</td>\n",
       "      <td>-5.6</td>\n",
       "      <td>-1.8</td>\n",
       "      <td>431.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>117.0</td>\n",
       "      <td>129.052264</td>\n",
       "      <td>1.965290</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-01-02 07:53:05</th>\n",
       "      <td>7.427107e-13</td>\n",
       "      <td>0.149933</td>\n",
       "      <td>0.164497</td>\n",
       "      <td>6.669903e-13</td>\n",
       "      <td>475231.043928</td>\n",
       "      <td>5.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1.3</td>\n",
       "      <td>141.0</td>\n",
       "      <td>-5.6</td>\n",
       "      <td>-1.8</td>\n",
       "      <td>431.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>117.0</td>\n",
       "      <td>129.052264</td>\n",
       "      <td>4.534402</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     Density_norm       FAC  FAC_power       Density  \\\n",
       "2015-01-02 00:50:50  5.932167e-13  0.104478   0.081088  6.468488e-13   \n",
       "2015-01-02 01:37:45  7.720531e-13  0.139248   0.138844  6.987142e-13   \n",
       "2015-01-02 02:24:40  5.577984e-13  0.113231   0.152365  6.089699e-13   \n",
       "2015-01-02 03:11:35  7.597623e-13  0.181961   0.320141  6.858343e-13   \n",
       "2015-01-02 03:58:30  5.691719e-13  0.075313   0.027412  6.237317e-13   \n",
       "2015-01-02 04:45:25  7.214728e-13  0.084554   0.050610  6.492501e-13   \n",
       "2015-01-02 05:32:20  5.458171e-13  0.090864   0.060029  5.960482e-13   \n",
       "2015-01-02 06:19:15  7.206711e-13  0.085828   0.040027  6.465145e-13   \n",
       "2015-01-02 07:06:10  5.683820e-13  0.048973   0.015818  6.189799e-13   \n",
       "2015-01-02 07:53:05  7.427107e-13  0.149933   0.164497  6.669903e-13   \n",
       "\n",
       "                          Altitude  Orbit_nr  Hemisphere  AP_INDEX   Kp  \\\n",
       "2015-01-02 00:50:50  464222.287865       1.0         1.0      15.0  3.0   \n",
       "2015-01-02 01:37:45  475492.964700       1.0        -1.0      15.0  3.0   \n",
       "2015-01-02 02:24:40  464224.157119       2.0         1.0      15.0  3.0   \n",
       "2015-01-02 03:11:35  475506.057772       2.0        -1.0       4.0  1.0   \n",
       "2015-01-02 03:58:30  464035.397442       3.0         1.0       4.0  1.0   \n",
       "2015-01-02 04:45:25  475514.819681       3.0        -1.0       4.0  1.0   \n",
       "2015-01-02 05:32:20  464242.140820       4.0         1.0       4.0  1.0   \n",
       "2015-01-02 06:19:15  475520.773958       4.0        -1.0       5.0  1.3   \n",
       "2015-01-02 07:06:10  464277.002808       5.0         1.0       5.0  1.3   \n",
       "2015-01-02 07:53:05  475231.043928       5.0        -1.0       5.0  1.3   \n",
       "\n",
       "                     F10_INDEX  BY_GSM  BZ_GSM  Bulk_speed   DST     AE  \\\n",
       "2015-01-02 00:50:50      141.0    -1.0    -2.1       433.0 -22.0  175.0   \n",
       "2015-01-02 01:37:45      141.0    -0.6    -1.7       433.0 -21.0  115.0   \n",
       "2015-01-02 02:24:40      141.0    -2.2    -2.4       434.0 -16.0  155.0   \n",
       "2015-01-02 03:11:35      141.0    -3.0    -2.3       433.0  -7.0   79.0   \n",
       "2015-01-02 03:58:30      141.0    -3.0    -2.3       433.0  -7.0   79.0   \n",
       "2015-01-02 04:45:25      141.0    -3.0    -1.7       429.0  -3.0   48.0   \n",
       "2015-01-02 05:32:20      141.0    -1.8    -1.1       416.0  -4.0   49.0   \n",
       "2015-01-02 06:19:15      141.0    -2.6    -1.1       416.0  -4.0   73.0   \n",
       "2015-01-02 07:06:10      141.0    -5.6    -1.8       431.0   1.0  117.0   \n",
       "2015-01-02 07:53:05      141.0    -5.6    -1.8       431.0   1.0  117.0   \n",
       "\n",
       "                       F10_Mean    PyAmps  \n",
       "2015-01-02 00:50:50  129.029218  1.769016  \n",
       "2015-01-02 01:37:45  129.032511  2.962752  \n",
       "2015-01-02 02:24:40  129.035803  1.880936  \n",
       "2015-01-02 03:11:35  129.039095  4.111182  \n",
       "2015-01-02 03:58:30  129.039095  1.841245  \n",
       "2015-01-02 04:45:25  129.042387  3.673843  \n",
       "2015-01-02 05:32:20  129.045679  1.109322  \n",
       "2015-01-02 06:19:15  129.048972  3.038630  \n",
       "2015-01-02 07:06:10  129.052264  1.965290  \n",
       "2015-01-02 07:53:05  129.052264  4.534402  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Half_orbit.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}