Sampling_FNIRS.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f01a1329",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "01d3d3a0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function to sample uniformly\n",
    "def uniform_sampling(group):\n",
    "    # Sort the group by seconds\n",
    "    group = group.sort_values(by='seconds')\n",
    "    \n",
    "    # Calculate the number of samples to take\n",
    "    num_samples = min(2000, len(group))\n",
    "    \n",
    "    # Sample uniformly spaced indices\n",
    "    indices = np.round(np.linspace(0, len(group) - 1, num_samples)).astype(int)\n",
    "    \n",
    "    # Select the rows using the sampled indices\n",
    "    sampled_data = group.iloc[indices]\n",
    "    \n",
    "    return sampled_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "d62124d0",
   "metadata": {},
   "outputs": [],
   "source": [
    "def np_encoder(object):\n",
    "    if isinstance(object, np.generic):\n",
    "        return object.item()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "179cda4c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load JSON data into a pandas DataFrame\n",
    "with open('ocarina_biomeasures_classified_hemodynamics_seconds.json', 'r') as file:\n",
    "    data = json.load(file)\n",
    "\n",
    "ocarina_df = pd.DataFrame(data)\n",
    "\n",
    "# Group by unique combinations of subject_id and trial_id\n",
    "ocarina_grouped = ocarina_df.groupby(['subject_id', 'trial_id'])\n",
    "\n",
    "\n",
    "# Apply uniform sampling to each group\n",
    "ocarina_sampled_df = ocarina_grouped.apply(uniform_sampling)\n",
    "\n",
    "# Reset index to remove hierarchical index\n",
    "ocarina_sampled_df.reset_index(drop=True, inplace=True)\n",
    "\n",
    "\n",
    "ocarina_sampled_df['subject_id'] = ocarina_sampled_df['subject_id'].astype(int)\n",
    "ocarina_sampled_df['trial_id'] = ocarina_sampled_df['trial_id'].astype(int)\n",
    "ocarina_sampled_df['timestamp'] = ocarina_sampled_df['timestamp'].astype(int)\n",
    "ocarina_sampled_df['workload_classification'] = ocarina_sampled_df['workload_classification'].astype(str)\n",
    "ocarina_sampled_df['workload_confidence'] = ocarina_sampled_df['workload_confidence'].astype(float)\n",
    "\n",
    "ocarina_sampled_df['attention_classification'] = ocarina_sampled_df['attention_classification'].astype(str)\n",
    "ocarina_sampled_df['attention_confidence'] = ocarina_sampled_df['attention_confidence'].astype(float)\n",
    "\n",
    "ocarina_sampled_df['perception_classification'] = ocarina_sampled_df['perception_classification'].astype(str)\n",
    "ocarina_sampled_df['perception_confidence'] = ocarina_sampled_df['perception_confidence'].astype(float)\n",
    "\n",
    "ocarina_sampled_df['seconds'] = ocarina_sampled_df['seconds'].astype(float)\n",
    "\n",
    "ocarina_sampled_df['workload_confidence'].fillna(0, inplace=True)\n",
    "ocarina_sampled_df['attention_confidence'].fillna(0, inplace=True)\n",
    "ocarina_sampled_df['perception_confidence'].fillna(0, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "1b69f90a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create the desired structure\n",
    "final_data = []\n",
    "for (subject_id, trial_id), group in ocarina_sampled_df.groupby(['subject_id', 'trial_id']):\n",
    "    obj = {\n",
    "        \"subject_id\": subject_id,\n",
    "        \"trial_id\": trial_id,\n",
    "        \"data\": group.drop(columns=['subject_id', 'trial_id']).to_dict(orient='records')\n",
    "    }\n",
    "    \n",
    "    \n",
    "    final_data.append(obj)\n",
    "\n",
    "# Output the final JSON structure to a file\n",
    "with open('FNIRS_sampled.json', 'w') as file:\n",
    "    json.dump(final_data, file, indent=4, default=np_encoder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "48f5b4b6",
   "metadata": {},
   "outputs": [],
   "source": [
    "gaze_df = pd.read_json('hl2_gaze_seconds.json')\n",
    "imu_df =  pd.read_json('hl2_imu_seconds.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "6320e6e2",
   "metadata": {},
   "outputs": [],
   "source": [
    "gaze_df_grouped = gaze_df.groupby(['subject_id', 'trial_id'])\n",
    "\n",
    "gaze_df_sampled = gaze_df_grouped.apply(uniform_sampling)\n",
    "\n",
    "gaze_df_sampled.reset_index(drop=True, inplace=True)\n",
    "\n",
    "gaze_df_sampled.drop(['hit_x', 'hit_y', 'hit_z'], inplace=True, axis=1)\n",
    "\n",
    "gaze_df_sampled.to_csv(\"gaze_sampled.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "22f2d096",
   "metadata": {},
   "outputs": [],
   "source": [
    "imu_df_grouped = imu_df.groupby(['subject_id', 'trial_id'])\n",
    "\n",
    "imu_df_sampled = imu_df_grouped.apply(uniform_sampling)\n",
    "\n",
    "imu_df_sampled.reset_index(drop=True, inplace=True)\n",
    "\n",
    "imu_df_sampled.drop(['orientation_covariance', 'angular_velocity_covariance', 'linear_acceleration_covariance', 'orientation_w' ], inplace=True, axis=1)\n",
    "\n",
    "imu_df_sampled.to_csv(\"imu_sampled.csv\", index= False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "ac95b15d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.00   -294.300018\n",
       "0.01   -285.150024\n",
       "0.05   -280.950012\n",
       "0.10   -279.000000\n",
       "0.25   -276.000000\n",
       "0.50   -273.150024\n",
       "0.75   -269.850006\n",
       "0.90   -266.700012\n",
       "0.95   -264.750000\n",
       "0.99   -260.400024\n",
       "1.00   -237.750015\n",
       "Name: orientation_z, dtype: float64"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "imu_df_sampled['orientation_z'].quantile([0, 0.01,0.05 ,0.10, 0.25,0.5,0.75, 0.90, 0.95, 0.99, 1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "ad9cf4bc",
   "metadata": {},
   "outputs": [],
   "source": [
    "#fix erroneous seconds\n",
    "mask = (gaze_df_sampled['subject_id'] == 8708) & (gaze_df_sampled['trial_id'] == 4)\n",
    "\n",
    "# Subtract 83979.895 only from the selected rows\n",
    "gaze_df_sampled.loc[mask, 'seconds'] -= 83979.895\n",
    "gaze_df_sampled.to_csv(\"gaze_sampled.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "9be47918",
   "metadata": {},
   "outputs": [],
   "source": [
    "#fix erroneous seconds\n",
    "mask = (imu_df_sampled['subject_id'] == 8708) & (imu_df_sampled['trial_id'] == 4)\n",
    "\n",
    "# Subtract 83979.895 only from the selected rows\n",
    "imu_df_sampled.loc[mask, 'seconds'] -= 83979.675\n",
    "imu_df_sampled.to_csv(\"imu_sampled.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "60a61f82",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}