NETICU_preprocessing_Python_V3.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Pipeline for BIAPT lab EEG Preprocessing: \n",
    "#### inspired by: https://github.com/hoechenberger/pybrain_mne/\n",
    "#### adapted by: Beatrice PDK, Victoria Sus and Charlotte Maschke, \n",
    "#### This pipeline uses MNE Python to preprocess EEG data: Plese go here: \n",
    "####                                https://mne.tools/stable/overview/index.html\n",
    "####  for more documentation on MNE Python"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Setup and import"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-07T21:44:24.521216Z",
     "start_time": "2022-02-07T21:44:23.539306Z"
    }
   },
   "outputs": [],
   "source": [
    "import matplotlib\n",
    "#import mne_bids\n",
    "import pathlib\n",
    "import mne\n",
    "import os\n",
    "import os.path as op\n",
    "from mne import viz\n",
    "import numpy as np\n",
    "import PyQt5\n",
    "# interactive plotting functions.\n",
    "\n",
    "\n",
    "from mne.preprocessing import (ICA, create_eog_epochs, create_ecg_epochs,corrmap)\n",
    "#import openneuro\n",
    "\n",
    "#from mne_bids import BIDSPath, read_raw_bids, print_dir_tree, make_report\n",
    "\n",
    "# Ensure Matplotlib uses the Qt5Agg backend, \n",
    "# which is the best choice for MNE-Python's \n",
    "# interactive plotting functions.\n",
    "matplotlib.use('Qt5Agg')\n",
    "%matplotlib qt\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Enter the recording information you want preprocess"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-07-13T15:09:37.685035Z",
     "start_time": "2022-07-13T15:09:37.664189Z"
    }
   },
   "outputs": [],
   "source": [
    "ID = \"030MW\"\n",
    "task = \"sedoff\" # \"sedon1\", \"sedoff\", \"sedon2\" \n",
    "save_task_as = \"sedoff\" # \"sedon1\", \"turn01\", \"sedoff\", \"turn02\", \"sedon2\"\n",
    "\n",
    "location = 'input location'\n",
    "output = 'output location'\n",
    "extension = 'mff'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-07-13T15:09:39.644017Z",
     "start_time": "2022-07-13T15:09:39.631509Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "raw_path = f\"{location}/sub-{ID}/eeg/sub-{ID}_task-{task}_eeg.{extension}\"\n",
    "raw_path"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load the raw data!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-07-13T15:09:44.929746Z",
     "start_time": "2022-07-13T15:09:44.867992Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "if extension == 'mff':\n",
    "    raw = mne.io.read_raw_egi(raw_path)\n",
    "if extension == 'set':\n",
    "    raw = mne.io.read_raw_eeglab(raw_path)\n",
    "raw"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# To crop (skip if no crop needed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "eeg_cropped = raw.crop(tmin = 690.1, tmax = 1364) # in seconds \n",
    "eeg_cropped"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Run if no crop needed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "eeg_cropped = raw.copy()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Resample the data to 250"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if eeg_cropped.info['sfreq'] != 250:\n",
    "    resampled = eeg_cropped.resample(250)\n",
    "else:\n",
    "    resampled = eeg_cropped.copy()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Keep the EEG only"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# this is to load EEG. If you want to load other stuff please refer to the website documentation\n",
    "eeg = resampled.pick_types(eeg = True)\n",
    "print('Number of channels in EEG:')\n",
    "len(eeg.ch_names)\n",
    "#eeg.ch_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# change the individual reference channel names to 'VREF' for all subjects (to avoid having different names for reference)\n",
    "if 'E129' in eeg.ch_names:\n",
    "    print('REF E129 replaced')\n",
    "    mne.rename_channels(eeg.info,{'E129':'VREF'})\n",
    "if 'E1001' in eeg.ch_names:\n",
    "    print('REF E1001 replaced')\n",
    "    mne.rename_channels(eeg.info,{'E1001':'VREF'})\n",
    "if 'Vertex Reference' in eeg.ch_names:\n",
    "    print('Vertex Reference replaced')\n",
    "    mne.rename_channels(eeg.info,{'Vertex Reference':'VREF'})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Apply filtering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-07T21:44:30.087141Z",
     "start_time": "2022-02-07T21:44:24.715216Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# load actual data into system (before it was only metadata)\n",
    "eeg.load_data()\n",
    "#eeg.load_data()\n",
    "\n",
    "# filter the data between 0.1 to 50 Hz\n",
    "eeg_filtered = eeg.filter(l_freq=0.1, h_freq = 50) # 0.1, 50\n",
    "\n",
    "# notch filter the data for freq =60\n",
    "eeg_notch = eeg_filtered.copy().notch_filter(freqs= 60)\n",
    "#eeg_notch = eeg_filtered.copy().notch_filter(freqs= 60, notch_widths = 2.25) # Default notch was not long enough for 027MW"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "%matplotlib qt\n",
    "viz.plot_raw_psd(eeg_notch, exclude = ['VREF'], fmax = 70)\n",
    "plt.title(ID)\n",
    "if not os.path.exists('{}/Preprocessing_info/sub-{}'.format(output,ID,save_task_as)) :\n",
    "    os.makedirs('{}/Preprocessing_info/sub-{}'.format(output,ID,save_task_as))\n",
    "plt.savefig('{}/Preprocessing_info/sub-{}/sub-{}_task-{}_PSD_raw_filtered.png'.\n",
    "            format(output,ID,ID,save_task_as))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Skip if not needed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#eeg_notch = eeg_notch.copy().notch_filter(freqs= 47) \n",
    "#eeg_notch = eeg_notch.copy().notch_filter(freqs= 41.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "%matplotlib qt\n",
    "viz.plot_raw_psd(eeg_notch, exclude = ['VREF'], fmax = 70)\n",
    "plt.title(ID)\n",
    "if not os.path.exists('{}/Preprocessing_info/sub-{}'.format(output,ID,save_task_as)) :\n",
    "    os.makedirs('{}/Preprocessing_info/sub-{}'.format(output,ID,save_task_as))\n",
    "plt.savefig('{}/Preprocessing_info/sub-{}/sub-{}_task-{}_PSD_raw_filtered2.png'.\n",
    "            format(output,ID,ID,save_task_as))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-07T20:51:37.136534Z",
     "start_time": "2022-02-07T20:51:37.126783Z"
    }
   },
   "source": [
    "## Visualize raw data to identify bad channels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "eeg_notch.plot(n_channels=60, duration=20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Verify if labelled correctly"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-07T21:44:30.445457Z",
     "start_time": "2022-02-07T21:44:30.442487Z"
    }
   },
   "outputs": [],
   "source": [
    "marked_bad = eeg_notch.info['bads']\n",
    "marked_bad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save in a txt\n",
    "if not os.path.exists('{}/Preprocessing_info/sub-{}'.format(output,ID)) :\n",
    "    os.makedirs('{}/Preprocessing_info/sub-{}'.format(output,ID))\n",
    "with open('{}/Preprocessing_info/sub-{}/sub-{}_task-{}_marked_bads.txt'.format(output,ID,ID,save_task_as), 'w') as outfile:\n",
    "    outfile.write(\"\\n\".join(marked_bad))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# just for plotting reasons, we add VREF here. This does not add VREF to the bad channels\n",
    "bads_tmp = eeg_notch.info['bads'].copy()\n",
    "bads_tmp.append('VREF')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib qt\n",
    "viz.plot_raw_psd(eeg_notch, exclude = bads_tmp, fmax = 70)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Remove bad channels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "eeg_badremoved = eeg_notch.copy().drop_channels(marked_bad)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Segment into 10-sec epochs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "epochs = mne.make_fixed_length_epochs(eeg_badremoved, duration = 10, overlap=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "epochs  #verify initial number"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Average Reference the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# use the average of all channels as reference\n",
    "eeg_avg_ref = epochs.load_data().set_eeg_reference(ref_channels='average')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Remove Non-Brain Electrodes "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-07T21:44:30.882892Z",
     "start_time": "2022-02-07T21:44:30.855951Z"
    }
   },
   "outputs": [],
   "source": [
    "non_brain_el = ['E127', 'E126', 'E17', 'E21', 'E14', 'E25', 'E8', 'E128', 'E125', 'E43', 'E120', 'E48', \n",
    "                'E119', 'E49', 'E113', 'E81', 'E73', 'E88', 'E68', 'E94', 'E63', 'E99', 'E56', 'E107' ]\n",
    "\n",
    "#only add non-brain channels if not already part of noisy channels\n",
    "for e in non_brain_el: \n",
    "    if e in eeg_avg_ref.info['ch_names']:\n",
    "        if e not in marked_bad :\n",
    "            eeg_avg_ref.info['bads'].append(e)\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-07T21:44:30.970116Z",
     "start_time": "2022-02-07T21:44:30.966748Z"
    }
   },
   "outputs": [],
   "source": [
    "print(eeg_avg_ref.info['bads'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# remove non-brain channels\n",
    "eeg_brainonly = eeg_avg_ref.copy().drop_channels(eeg_avg_ref.info['bads'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Reject epochs with amplitude bigger than 2000 µVolt "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Peak to peak amplitude on brain scalp > 2000 µVolt are epochs not linked with physiological causes, physiological amplitude accepted < 800 µVolt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "epochs_clean = eeg_brainonly.copy().load_data()\n",
    "#epochs_clean.drop_bad({'eeg':600*1e-6})\n",
    "#epochs_clean.plot_drop_log()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "epochs_clean.plot(title='bad_epochs_remaining', n_epochs=3, n_channels=120, scalings=20e-6)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Check final data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#eeg_brainonly.plot(n_epochs=3, n_channels=100, scalings=20e-6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "epochs_clean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-02-07T21:44:30.445457Z",
     "start_time": "2022-02-07T21:44:30.442487Z"
    }
   },
   "outputs": [],
   "source": [
    "# marked_bad = epochs_clean.info['bads']\n",
    "# marked_bad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# epochs_clean = epochs_clean.copy().drop_channels(marked_bad)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "%matplotlib qt\n",
    "epochs_clean.plot_psd(fmax=70, exclude = ['VREF'])\n",
    "plt.title(ID)\n",
    "if not os.path.exists('{}/Preprocessing_info/sub-{}'.format(output,ID,save_task_as)) :\n",
    "    os.makedirs('{}/Preprocessing_info/sub-{}'.format(output,ID,save_task_as))\n",
    "plt.savefig('{}/Preprocessing_info/sub-{}/sub-{}_task-{}_PSD_final.png'.\n",
    "            format(output,ID,ID,save_task_as))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save a list of dropped epochs\n",
    "dropped_epochs =  [n for n, dl in enumerate(epochs_clean.drop_log) if len(dl)]\n",
    "dropped_epochs = [str(i) for i in dropped_epochs] # need to convert to str to save\n",
    "# save in a txt\n",
    "if not os.path.exists('{}/Preprocessing_info/sub-{}'.format(output,ID)) :\n",
    "    os.makedirs('{}/Preprocessing_info/sub-{}'.format(output,ID))\n",
    "with open('{}/Preprocessing_info/sub-{}/sub-{}_task-{}_dropped_epochs.txt'.format(output,ID,ID,save_task_as), 'w') as outfile:\n",
    "    outfile.write(\"\\n\".join(dropped_epochs))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save final brain data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get some summary info\n",
    "output_summary = {\n",
    "    \"nr_epochs_final\": len(epochs_clean),\n",
    "    \"nr_epochs_dropped\": len(epochs)-len(epochs_clean),\n",
    "    \"nr_channels_final\": len(epochs_clean.info['ch_names']),\n",
    "    \"nr_channels_dropped\": 129 - len(non_brain_el)-len(epochs_clean.info['ch_names'])}\n",
    "\n",
    "with open('{}/Preprocessing_info/sub-{}/sub-{}_task-{}_SUMMARY.txt'.format(output,ID,ID,save_task_as), 'w') as file:\n",
    "    print(output_summary, file=file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not os.path.exists(f'{output}/sub-{ID}/eeg/') :\n",
    "    os.makedirs(f'/{output}/sub-{ID}/eeg/')\n",
    "\n",
    "epochs_clean.save(f\"/{output}/sub-{ID}/eeg/sub-{ID}_task-{save_task_as}_epoch_eeg.fif\", overwrite=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('End :)')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "35cfb6b982167ba7388688fc86c2f436cc5633f53a9734623cf65ff864c3b3fd"
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.17"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}