Data_init2.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Get Omni data and prepare for normalization factors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pickle\n",
    "from spacepy import pycdf\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import datetime"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "load the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "cdf = pycdf.CDF('/home/simon/Desktop/Bachelor_project/data/omni2_h0_mrg1hr_20150101_v01.cdf')\n",
    "data = cdf.copy()\n",
    "cdf.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['Epoch', 'YR', 'Day', 'HR', 'Rot#', 'IMF', 'PLS', 'IMF_PTS', 'PLS_PTS', 'ABS_B', 'F', 'THETA_AV', 'PHI_AV', 'BX_GSE', 'BY_GSE', 'BZ_GSE', 'BY_GSM', 'BZ_GSM', 'SIGMA-ABS_B', 'SIGMA-B', 'SIGMA-Bx', 'SIGMA-By', 'SIGMA-Bz', 'T', 'N', 'V', 'PHI-V', 'THETA-V', 'Ratio', 'Pressure', 'SIGMA-T', 'SIGMA-N', 'SIGMA-V', 'SIGMA-PHI-V', 'SIGMA-THETA-V', 'SIGMA-ratio', 'E', 'Beta', 'Mach_num', 'Mgs_mach_num', 'PR-FLX_1', 'PR-FLX_2', 'PR-FLX_4', 'PR-FLX_10', 'PR-FLX_30', 'PR-FLX_60', 'MFLX', 'R', 'F10_INDEX', 'KP', 'DST', 'AE', 'AP_INDEX', 'AL_INDEX', 'AU_INDEX', 'PC_N_INDEX', 'Solar_Lyman_alpha'])"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.keys()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Get the time "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "start =datetime.datetime(data['YR'][0],1,1,data['HR'][0])\n",
    "dates = start + (data['Day']-1) * datetime.timedelta(days=1)+(data['HR']) *60* datetime.timedelta(minutes=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create data frame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "selected_data= [np.array(data['AP_INDEX']),np.array(data['KP'])/10,np.array(data['F10_INDEX'])\n",
    "                ,np.array(data['BY_GSM']),np.array(data['BZ_GSM']),np.array(data['V']),np.array(data['DST']),np.array(data['AE'])]\n",
    "names = ['AP_INDEX','Kp','F10_INDEX','BY_GSM','BZ_GSM','Bulk_speed','DST','AE']\n",
    "df = pd.DataFrame(np.transpose(selected_data), index=dates, columns=names)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Add 81-day mean of F10.7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "selected_data= [np.array(data['F10_INDEX'])]\n",
    "names = ['F10_Mean']\n",
    "F10_Mean = pd.DataFrame(np.transpose(selected_data), index=dates, columns=names)\n",
    "F10_Mean=pd.concat([F10_Mean.F10_Mean,df.F10_INDEX])\n",
    "F10_Mean=F10_Mean.rolling(81*24,min_periods=81*24).mean().dropna(how='any')\n",
    "df.loc[:,'F10_Mean'] = F10_Mean[df.index[0]:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_pickle('/home/simon/Desktop/Bachelor_project/data/pkl_files/omni.pkl')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prepare for density corrections i matlab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "DNS = pd.read_pickle('/home/simon/Desktop/Bachelor_project/data/pkl_files/DNS.pkl')\n",
    "df = df.loc[:,['AP_INDEX','Kp','F10_INDEX','F10_Mean']]\n",
    "omni = df.reindex(DNS.index,method='ffill')\n",
    "DNS = pd.concat([DNS,omni], axis=1)\n",
    "DNS.loc[:,'dayofyear'] = DNS.index.dayofyear\n",
    "DNS.loc[:,'UTseconds'] = (DNS.index.hour*60+DNS.index.minute)*60+DNS.index.second\n",
    "DNS.to_csv('/home/simon/Desktop/Bachelor_project/data/DNS_matlab.csv',index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}