notebooks/2020-07-07_compute-outliers-and-median-motion.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from glob import glob\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "subjects = [s.split('/')[-1] for s in sorted(glob('../data/sub-*'))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_confounds(subject):\n",
    "    confounds = sorted(glob(f\"../outputs/fmriprep/{subject}/func/*.tsv\"))\n",
    "    dfs = [pd.read_csv(c, sep='\\t') for c in confounds]\n",
    "    return dfs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_cols(df):\n",
    "    cols = ['framewise_displacement']\n",
    "    cols += [c for c in df.columns if 'motion_outlier' in c]\n",
    "    return df[cols]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_outliers = []\n",
    "framewise_displacement = []\n",
    "for subj in subjects:\n",
    "    dfs = load_confounds(subj)\n",
    "    dfs = [extract_cols(d) for d in dfs]\n",
    "    for df in dfs:\n",
    "        # add framewise displacement to a list since we're just going to compute\n",
    "        # a group metric (median, min, max)\n",
    "        framewise_displacement += df['framewise_displacement'].values.tolist()\n",
    "        n_outliers.append(len([c for c in df.columns if 'motion' in c]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# split outliers into subjects again\n",
    "n_outliers_subject = np.split(np.array(n_outliers), len(subjects))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sub-sid000005: 26\n",
      "sub-sid000007: 116\n",
      "sub-sid000009: 695\n",
      "sub-sid000010: 139\n",
      "sub-sid000013: 97\n",
      "sub-sid000020: 28\n",
      "sub-sid000021: 32\n",
      "sub-sid000024: 35\n",
      "sub-sid000025: 1\n",
      "sub-sid000029: 299\n",
      "sub-sid000030: 114\n",
      "sub-sid000034: 178\n",
      "sub-sid000050: 115\n",
      "sub-sid000052: 318\n",
      "sub-sid000055: 111\n",
      "sub-sid000114: 56\n",
      "sub-sid000120: 37\n",
      "sub-sid000134: 73\n",
      "sub-sid000142: 30\n",
      "sub-sid000278: 118\n",
      "sub-sid000416: 83\n",
      "sub-sid000499: 56\n",
      "sub-sid000522: 25\n",
      "sub-sid000535: 11\n",
      "sub-sid000560: 223\n"
     ]
    }
   ],
   "source": [
    "for subj, out in zip(subjects, n_outliers_subject):\n",
    "    print(f\"{subj}: {out.sum()}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfs = load_confounds('sub-sid000009')\n",
    "# get number of TRs for each run so we can compute percentage\n",
    "n_trs = [len(df) for df in dfs]\n",
    "all_n_trs = np.sum(n_trs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sub-sid000005: 0.85 %\n",
      "sub-sid000007: 3.80 %\n",
      "sub-sid000009: 22.77 %\n",
      "sub-sid000010: 4.55 %\n",
      "sub-sid000013: 3.18 %\n",
      "sub-sid000020: 0.92 %\n",
      "sub-sid000021: 1.05 %\n",
      "sub-sid000024: 1.15 %\n",
      "sub-sid000025: 0.03 %\n",
      "sub-sid000029: 9.80 %\n",
      "sub-sid000030: 3.74 %\n",
      "sub-sid000034: 5.83 %\n",
      "sub-sid000050: 3.77 %\n",
      "sub-sid000052: 10.42 %\n",
      "sub-sid000055: 3.64 %\n",
      "sub-sid000114: 1.83 %\n",
      "sub-sid000120: 1.21 %\n",
      "sub-sid000134: 2.39 %\n",
      "sub-sid000142: 0.98 %\n",
      "sub-sid000278: 3.87 %\n",
      "sub-sid000416: 2.72 %\n",
      "sub-sid000499: 1.83 %\n",
      "sub-sid000522: 0.82 %\n",
      "sub-sid000535: 0.36 %\n",
      "sub-sid000560: 7.31 %\n"
     ]
    }
   ],
   "source": [
    "for subj, out in zip(subjects, n_outliers_subject):\n",
    "    print(f\"{subj}: {out.sum()/all_n_trs*100:.2f} %\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "percentage_outliers = np.array([out.sum()/all_n_trs*100 for out in n_outliers_subject])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Compute the median percentage of outliers, min, and max"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 2.72,  0.03, 22.77])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.round((np.median(percentage_outliers), np.min(percentage_outliers), np.max(percentage_outliers)), 2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "How many subjects with less than 5% outliers?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "20"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.sum(percentage_outliers < 5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now compute median framewise dispalcement, max, and min"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "framewise_displacement_subjects = np.split(np.array(framewise_displacement), len(subjects))\n",
    "median_fd_subjects = [np.nanmedian(fd) for fd in framewise_displacement_subjects]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.09, 0.06, 0.19])"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.round((np.nanmedian(median_fd_subjects), np.nanmin(median_fd_subjects), np.nanmax(median_fd_subjects)), 2)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}