IBM SparkML Final Project_randomforest.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "f6b4118b-bbba-4418-9bae-2a2c4fffa660",
   "metadata": {
    "papermill": {
     "duration": 0.005346,
     "end_time": "2021-03-17T17:21:55.111968",
     "exception": false,
     "start_time": "2021-03-17T17:21:55.106622",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Clone Claimed from Github"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "12d06cfb-19dc-4651-8819-62746543fb4c",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Cloning into 'claimed-component-library'...\n",
      "Checking out files: 100% (434/434), done.\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "rm -Rf component-library\n",
    "rm -Rf claimed\n",
    "rm -Rf claimed-component-library\n",
    "rm -Rf data\n",
    "rm -Rf HMP_Dataset\n",
    "git clone https://github.com/WazirRohiman/claimed-component-library.git"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7b6807d2-5c7c-4ecc-ab1a-91e5fce97b6c",
   "metadata": {
    "papermill": {
     "duration": 0.005346,
     "end_time": "2021-03-17T17:21:55.111968",
     "exception": false,
     "start_time": "2021-03-17T17:21:55.106622",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Python Version Check"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "550b2b5b-83c9-4326-b6db-1fa24b4b9179",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "37\n",
      "Starting installation...\n",
      "Please <<RESTART YOUR KERNEL>> (Kernel->Restart Kernel and Clear All Outputs)\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "export version=`python --version |awk '{print $2}' |awk -F\".\" '{print $1$2}'`\n",
    "\n",
    "echo $version\n",
    "\n",
    "if [ $version == '36' ] || [ $version == '37' ]; then\n",
    "    echo 'Starting installation...'\n",
    "    pip3 install pyspark==2.4.8 wget==3.2 > install.log 2> install.log\n",
    "    if [ $? == 0 ]; then\n",
    "        echo 'Please <<RESTART YOUR KERNEL>> (Kernel->Restart Kernel and Clear All Outputs)'\n",
    "    else\n",
    "        echo 'Installation failed, please check log:'\n",
    "        cat install.log\n",
    "    fi\n",
    "elif [ $version == '38' ] || [ $version == '39' ]; then\n",
    "    pip3 install pyspark==3.1.2 wget==3.2 > install.log 2> install.log\n",
    "    if [ $? == 0 ]; then\n",
    "        echo 'Please <<RESTART YOUR KERNEL>> (Kernel->Restart Kernel and Clear All Outputs)'\n",
    "    else\n",
    "        echo 'Installation failed, please check log:'\n",
    "        cat install.log\n",
    "    fi\n",
    "else\n",
    "    echo 'Currently only python 3.6, 3.7 , 3.8 and 3.9 are supported, in case you need a different version please open an issue at https://github.com/IBM/claimed/issues'\n",
    "    exit -1\n",
    "fi"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5bfcd414-7136-4af0-ba71-7553a8a64e33",
   "metadata": {
    "papermill": {
     "duration": 0.005346,
     "end_time": "2021-03-17T17:21:55.111968",
     "exception": false,
     "start_time": "2021-03-17T17:21:55.106622",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Python imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "d3376d0d-e16a-4a97-9b0a-ebd80c0e9133",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception ignored in: <function JavaWrapper.__del__ at 0x7f9d58c46560>\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/jupyterlab/conda/envs/python/lib/python3.7/site-packages/pyspark/ml/wrapper.py\", line 40, in __del__\n",
      "    if SparkContext._active_spark_context and self._java_obj is not None:\n",
      "AttributeError: 'VectorAssembler' object has no attribute '_java_obj'\n"
     ]
    }
   ],
   "source": [
    "import fnmatch\n",
    "import os\n",
    "import pandas as pd\n",
    "from pathlib import Path\n",
    "from pyspark import SparkConf\n",
    "from pyspark import SparkContext\n",
    "from pyspark import SQLContext\n",
    "from pyspark.sql.functions import lit\n",
    "from pyspark.sql import SparkSession\n",
    "from pyspark.sql.types import IntegerType\n",
    "from pyspark.sql.types import StructField\n",
    "from pyspark.sql.types import StructType\n",
    "import random\n",
    "import re\n",
    "import shutil\n",
    "import sys\n",
    "import logging\n",
    "import glob\n",
    "import site\n",
    "import wget\n",
    "\n",
    "from pyspark.ml.classification import RandomForestClassifier\n",
    "from pyspark.ml import Pipeline\n",
    "from pyspark.ml.evaluation import MulticlassClassificationEvaluator\n",
    "from pyspark.ml.feature import StringIndexer\n",
    "from pyspark.ml.feature import VectorAssembler\n",
    "from pyspark.ml.feature import MinMaxScaler\n",
    "\n",
    "from pyspark.sql.types import DoubleType\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "022a8818-d3a9-467f-b832-e8df7e1eea5d",
   "metadata": {
    "papermill": {
     "duration": 0.005346,
     "end_time": "2021-03-17T17:21:55.111968",
     "exception": false,
     "start_time": "2021-03-17T17:21:55.106622",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Input HMP data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "70a1b32d-e759-4279-a0fc-7acb8a31a231",
   "metadata": {
    "papermill": {
     "duration": 0.005346,
     "end_time": "2021-03-17T17:21:55.111968",
     "exception": false,
     "start_time": "2021-03-17T17:21:55.106622",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Setting paths and file names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "3deb6e28-18cd-4d55-a80b-ed4a310da46e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# path and file name for output (default: data.csv)\n",
    "data_csv = os.environ.get('data_csv', 'data.csv')\n",
    "\n",
    "# url of master (default: local mode)\n",
    "master = os.environ.get('master', \"local[*]\")\n",
    "\n",
    "# temporal data storage for local execution\n",
    "data_dir = os.environ.get('data_dir', '../../data/')\n",
    "\n",
    "# sample on input data to increase processing speed 0..1 (default: 1.0)\n",
    "sample = os.environ.get('sample', '1.0')\n",
    "\n",
    "# destination path and parquet file name (default: data.parquet)\n",
    "output_data_parquet = os.environ.get('output_data_parquet', 'data.parquet')\n",
    "\n",
    "# data_parquet path and parquet file name (default: data.parquet)\n",
    "data_parquet = os.environ.get('data_parquet', 'data.parquet')\n",
    "\n",
    "# target condensed parquet file (default: data_condensed.parquet)\n",
    "data_parquet_condensed = os.environ.get('data_parquet', 'data_condensed.parquet')\n",
    "\n",
    "# target condensed parquet file (default: data_condensed.parquet)\n",
    "data_csv_condensed = os.environ.get('data_csv', 'data_condensed.csv')\n",
    "\n",
    "#for random forest\n",
    "model_target = os.environ.get('model_target', \"model.xml\")  # model output file name\n",
    "\n",
    "input_columns = os.environ.get('input_columns', '[\"x\", \"y\", \"z\"]')  # input columns to consider\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ceb26a0d-dc7b-44bf-bc7e-b90b38844015",
   "metadata": {},
   "source": [
    "Parameters for HMP input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "f26ad1aa-8789-4af3-a6ef-e592f34a783a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# override parameters received from a potential call using %run magic\n",
    "parameters = list(\n",
    "    map(\n",
    "        lambda s: re.sub('$', '\"', s),\n",
    "        map(\n",
    "            lambda s: s.replace('=', '=\"'),\n",
    "            filter(\n",
    "                lambda s: s.find('=') > -1,\n",
    "                sys.argv\n",
    "            )\n",
    "        )\n",
    "    )\n",
    ")\n",
    "\n",
    "for parameter in parameters:\n",
    "    exec(parameter)\n",
    "\n",
    "# cast parameters to appropriate type\n",
    "sample = float(sample)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e967f546-900c-4d00-8fa2-575d77bbc2d2",
   "metadata": {
    "papermill": {
     "duration": 0.005346,
     "end_time": "2021-03-17T17:21:55.111968",
     "exception": false,
     "start_time": "2021-03-17T17:21:55.106622",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Create Spark context"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "01bef351-1733-41a9-bc5d-2ce4a8f95672",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "23/03/14 05:24:45 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n",
      "Setting default log level to \"WARN\".\n",
      "To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n"
     ]
    }
   ],
   "source": [
    "sc = SparkContext \\\n",
    "    .getOrCreate(SparkConf() \\\n",
    "    .setMaster(master))\n",
    "\n",
    "spark = SparkSession \\\n",
    "    .builder \\\n",
    "    .getOrCreate()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1ea727cc-3d52-4350-92f1-0918d12e74e1",
   "metadata": {},
   "source": [
    "Pull the data in raw format from the source (github)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "5fd055de-f6e3-4a83-9cd2-bdb54225f358",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cloning into 'HMP_Dataset'...\n",
      "remote: Enumerating objects: 865, done.\u001b[K\n",
      "remote: Total 865 (delta 0), reused 0 (delta 0), pack-reused 865\u001b[K\n",
      "Receiving objects: 100% (865/865), 1010.96 KiB | 267.00 KiB/s, done.\n",
      "Checking out files: 100% (848/848), done.\n"
     ]
    }
   ],
   "source": [
    "!rm -Rf HMP_Dataset\n",
    "!git clone https://github.com/wchill/HMP_Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "f8abbcc3-45e5-4b41-9ccf-85c1ddbe9ef1",
   "metadata": {},
   "outputs": [],
   "source": [
    "schema = StructType([\n",
    "    StructField(\"x\", IntegerType(), True),\n",
    "    StructField(\"y\", IntegerType(), True),\n",
    "    StructField(\"z\", IntegerType(), True)])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c1288c8c-167b-4c93-bb4e-031930f55e8d",
   "metadata": {},
   "source": [
    "This step takes a while, it parses through all files and folders and creates a temporary dataframe for each file which gets appended to an overall data-frame \"df\". In addition, a column called \"class\" is added to allow for straightforward usage in Spark afterwards in a supervised machine learning scenario for example."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "1c86ec7f-e255-4700-a27a-b642ddd3b3d6",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accelerometer-2011-04-11-13-28-18-brush_teeth-f1.txt\n",
      "Accelerometer-2011-04-11-13-29-54-brush_teeth-f1.txt\n",
      "Accelerometer-2011-05-30-08-35-11-brush_teeth-f1.txt\n",
      "Accelerometer-2011-05-30-09-36-50-brush_teeth-f1.txt\n",
      "Accelerometer-2011-05-30-10-34-16-brush_teeth-m1.txt\n",
      "Accelerometer-2011-05-30-21-10-57-brush_teeth-f1.txt\n",
      "Accelerometer-2011-05-30-21-55-04-brush_teeth-m2.txt\n",
      "Accelerometer-2011-05-31-15-16-47-brush_teeth-f1.txt\n",
      "Accelerometer-2011-06-02-10-42-22-brush_teeth-f1.txt\n",
      "Accelerometer-2011-06-02-10-45-50-brush_teeth-f1.txt\n",
      "Accelerometer-2011-06-06-10-45-27-brush_teeth-f1.txt\n",
      "Accelerometer-2011-06-06-10-48-05-brush_teeth-f1.txt\n",
      "Accelerometer-2011-03-24-10-24-39-climb_stairs-f1.txt\n",
      "Accelerometer-2011-03-24-10-25-44-climb_stairs-f1.txt\n",
      "Accelerometer-2011-03-29-09-55-46-climb_stairs-f1.txt\n",
      "Accelerometer-2011-04-05-18-21-22-climb_stairs-f1.txt\n",
      "Accelerometer-2011-04-05-18-32-29-climb_stairs-f1.txt\n",
      "Accelerometer-2011-04-11-11-44-35-climb_stairs-f1.txt\n",
      "Accelerometer-2011-04-11-11-57-50-climb_stairs-f1.txt\n",
      "Accelerometer-2011-04-11-11-58-30-climb_stairs-f1.txt\n",
      "Accelerometer-2011-05-30-08-21-38-climb_stairs-f1.txt\n",
      "Accelerometer-2011-05-30-08-30-37-climb_stairs-f1.txt\n",
      "Accelerometer-2011-05-30-08-39-09-climb_stairs-f1.txt\n",
      "Accelerometer-2011-05-30-09-29-58-climb_stairs-f1.txt\n",
      "Accelerometer-2011-05-30-09-38-52-climb_stairs-f1.txt\n",
      "Accelerometer-2011-05-30-09-43-37-climb_stairs-f1.txt\n",
      "Accelerometer-2011-05-30-10-20-47-climb_stairs-m1.txt\n",
      "Accelerometer-2011-05-30-10-32-53-climb_stairs-m1.txt\n",
      "Accelerometer-2011-05-30-10-41-10-climb_stairs-m1.txt\n",
      "Accelerometer-2011-05-30-20-54-39-climb_stairs-f1.txt\n",
      "Accelerometer-2011-05-30-21-02-43-climb_stairs-f1.txt\n",
      "Accelerometer-2011-05-30-21-14-55-climb_stairs-f1.txt\n",
      "Accelerometer-2011-05-30-21-36-58-climb_stairs-m2.txt\n",
      "Accelerometer-2011-05-30-21-46-05-climb_stairs-m2.txt\n",
      "Accelerometer-2011-05-30-22-00-32-climb_stairs-m2.txt\n",
      "Accelerometer-2011-05-31-14-58-41-climb_stairs-f1.txt\n",
      "Accelerometer-2011-05-31-15-06-32-climb_stairs-f1.txt\n",
      "Accelerometer-2011-05-31-15-20-50-climb_stairs-f1.txt\n",
      "Accelerometer-2011-05-31-16-26-09-climb_stairs-f1.txt\n",
      "Accelerometer-2011-05-31-16-32-44-climb_stairs-f1.txt\n",
      "Accelerometer-2011-06-02-10-37-15-climb_stairs-f1.txt\n",
      "Accelerometer-2011-06-02-10-37-44-climb_stairs-f1.txt\n",
      "Accelerometer-2011-06-02-10-39-21-climb_stairs-f1.txt\n",
      "Accelerometer-2011-06-02-10-40-10-climb_stairs-f1.txt\n",
      "Accelerometer-2011-06-06-10-57-19-climb_stairs-f1.txt\n",
      "Accelerometer-2011-06-06-10-57-43-climb_stairs-f1.txt\n",
      "Accelerometer-2011-06-06-10-59-08-climb_stairs-f1.txt\n",
      "Accelerometer-2011-06-06-10-59-58-climb_stairs-f1.txt\n",
      "Accelerometer-2011-06-06-11-00-50-climb_stairs-f1.txt\n",
      "Accelerometer-2011-06-06-11-01-27-climb_stairs-f1.txt\n",
      "Accelerometer-2011-06-06-11-02-24-climb_stairs-f1.txt\n",
      "Accelerometer-2012-05-28-17-56-03-climb_stairs-m1.txt\n",
      "Accelerometer-2012-05-28-17-56-40-climb_stairs-m1.txt\n",
      "Accelerometer-2012-05-29-16-53-12-climb_stairs-f2.txt\n",
      "Accelerometer-2012-05-29-16-54-38-climb_stairs-f2.txt\n",
      "Accelerometer-2012-05-29-17-17-05-climb_stairs-m3.txt\n",
      "Accelerometer-2012-05-29-17-17-51-climb_stairs-m3.txt\n",
      "Accelerometer-2012-05-30-18-31-08-climb_stairs-f3.txt\n",
      "Accelerometer-2012-05-30-18-31-52-climb_stairs-f3.txt\n",
      "Accelerometer-2012-05-30-18-32-56-climb_stairs-f3.txt\n",
      "Accelerometer-2012-05-30-18-33-57-climb_stairs-f3.txt\n",
      "Accelerometer-2012-05-30-19-12-19-climb_stairs-m4.txt\n",
      "Accelerometer-2012-05-30-19-12-57-climb_stairs-m4.txt\n",
      "Accelerometer-2012-05-30-19-13-50-climb_stairs-m4.txt\n",
      "Accelerometer-2012-05-30-19-14-51-climb_stairs-m4.txt\n",
      "Accelerometer-2012-05-30-22-06-17-climb_stairs-m2.txt\n",
      "Accelerometer-2012-05-30-22-06-47-climb_stairs-m2.txt\n",
      "Accelerometer-2012-05-30-22-07-15-climb_stairs-m2.txt\n",
      "Accelerometer-2012-05-30-22-07-44-climb_stairs-m2.txt\n",
      "Accelerometer-2012-05-30-22-08-15-climb_stairs-m2.txt\n",
      "Accelerometer-2012-06-06-08-53-29-climb_stairs-m5.txt\n",
      "Accelerometer-2012-06-06-08-53-47-climb_stairs-m5.txt\n",
      "Accelerometer-2012-06-06-08-54-08-climb_stairs-m5.txt\n",
      "Accelerometer-2012-06-06-08-54-29-climb_stairs-m5.txt\n",
      "Accelerometer-2012-06-06-08-59-53-climb_stairs-m5.txt\n",
      "Accelerometer-2012-06-06-09-00-24-climb_stairs-m5.txt\n",
      "Accelerometer-2012-06-06-09-00-51-climb_stairs-m5.txt\n",
      "Accelerometer-2012-06-06-09-01-08-climb_stairs-m5.txt\n",
      "Accelerometer-2012-06-06-09-05-28-climb_stairs-m5.txt\n",
      "Accelerometer-2012-06-06-09-06-03-climb_stairs-m5.txt\n",
      "Accelerometer-2012-06-06-09-06-28-climb_stairs-m5.txt\n",
      "Accelerometer-2012-06-06-09-06-49-climb_stairs-m5.txt\n",
      "Accelerometer-2012-06-06-09-32-04-climb_stairs-m6.txt\n",
      "Accelerometer-2012-06-06-09-32-36-climb_stairs-m6.txt\n",
      "Accelerometer-2012-06-06-09-32-58-climb_stairs-m6.txt\n",
      "Accelerometer-2012-06-06-09-33-15-climb_stairs-m6.txt\n",
      "Accelerometer-2012-06-06-09-37-23-climb_stairs-m6.txt\n",
      "Accelerometer-2012-06-06-09-37-58-climb_stairs-m6.txt\n",
      "Accelerometer-2012-06-06-09-38-17-climb_stairs-m6.txt\n",
      "Accelerometer-2012-06-06-09-38-43-climb_stairs-m6.txt\n",
      "Accelerometer-2012-06-06-09-42-55-climb_stairs-m6.txt\n",
      "Accelerometer-2012-06-06-09-43-14-climb_stairs-m6.txt\n",
      "Accelerometer-2012-06-06-09-43-47-climb_stairs-m6.txt\n",
      "Accelerometer-2012-06-06-09-44-02-climb_stairs-m6.txt\n",
      "Accelerometer-2012-06-06-14-04-18-climb_stairs-m7.txt\n",
      "Accelerometer-2012-06-06-14-04-51-climb_stairs-m7.txt\n",
      "Accelerometer-2012-06-06-14-05-05-climb_stairs-m7.txt\n",
      "Accelerometer-2012-06-06-14-05-18-climb_stairs-m7.txt\n",
      "Accelerometer-2012-06-06-14-08-44-climb_stairs-m7.txt\n",
      "Accelerometer-2012-06-06-14-08-57-climb_stairs-m7.txt\n",
      "Accelerometer-2012-06-06-14-09-12-climb_stairs-m7.txt\n",
      "Accelerometer-2012-06-06-14-09-24-climb_stairs-m7.txt\n",
      "Accelerometer-2012-06-06-14-12-35-climb_stairs-m7.txt\n",
      "Accelerometer-2012-06-06-14-12-49-climb_stairs-m7.txt\n",
      "Accelerometer-2012-06-06-14-13-05-climb_stairs-m7.txt\n",
      "Accelerometer-2012-06-06-14-13-20-climb_stairs-m7.txt\n",
      "Accelerometer-2012-06-07-10-48-08-climb_stairs-f1.txt\n",
      "Accelerometer-2012-06-07-10-48-24-climb_stairs-f1.txt\n",
      "Accelerometer-2012-06-07-10-49-17-climb_stairs-f1.txt\n",
      "Accelerometer-2012-06-07-10-49-35-climb_stairs-f1.txt\n",
      "Accelerometer-2012-06-07-10-53-34-climb_stairs-f1.txt\n",
      "Accelerometer-2012-06-07-10-53-53-climb_stairs-f1.txt\n",
      "Accelerometer-2012-06-07-10-54-12-climb_stairs-f1.txt\n",
      "Accelerometer-2012-06-07-10-54-29-climb_stairs-f1.txt\n",
      "Accelerometer-2011-03-24-09-44-34-comb_hair-f1.txt\n",
      "Accelerometer-2011-03-24-10-26-33-comb_hair-f1.txt\n",
      "Accelerometer-2011-03-24-10-57-40-comb_hair-f1.txt\n",
      "Accelerometer-2011-03-24-16-10-36-comb_hair-f2.txt\n",
      "Accelerometer-2011-03-29-08-57-24-comb_hair-f1.txt\n",
      "Accelerometer-2011-04-11-13-31-24-comb_hair-f1.txt\n",
      "Accelerometer-2011-05-30-08-32-58-comb_hair-f1.txt\n",
      "Accelerometer-2011-05-30-09-39-49-comb_hair-f1.txt\n",
      "Accelerometer-2011-05-30-10-36-26-comb_hair-m1.txt\n",
      "Accelerometer-2011-05-30-21-08-40-comb_hair-f1.txt\n",
      "Accelerometer-2011-05-30-21-57-03-comb_hair-m2.txt\n",
      "Accelerometer-2011-05-31-15-14-55-comb_hair-f1.txt\n",
      "Accelerometer-2011-06-02-10-41-33-comb_hair-f1.txt\n",
      "Accelerometer-2011-06-02-10-44-17-comb_hair-f1.txt\n",
      "Accelerometer-2011-06-02-16-12-44-comb_hair-f1.txt\n",
      "Accelerometer-2011-06-02-16-14-07-comb_hair-f1.txt\n",
      "Accelerometer-2011-06-02-16-18-54-comb_hair-f1.txt\n",
      "Accelerometer-2011-06-02-16-20-04-comb_hair-f1.txt\n",
      "Accelerometer-2011-06-02-16-48-41-comb_hair-f4.txt\n",
      "Accelerometer-2011-06-02-16-49-27-comb_hair-f4.txt\n",
      "Accelerometer-2011-06-02-16-56-03-comb_hair-f4.txt\n",
      "Accelerometer-2011-06-02-17-02-50-comb_hair-f4.txt\n",
      "Accelerometer-2011-06-02-17-03-33-comb_hair-f4.txt\n",
      "Accelerometer-2011-06-02-17-18-58-comb_hair-m1.txt\n",
      "Accelerometer-2011-06-02-17-32-57-comb_hair-m1.txt\n",
      "Accelerometer-2011-06-02-17-39-02-comb_hair-m1.txt\n",
      "Accelerometer-2011-06-02-17-39-54-comb_hair-m1.txt\n",
      "Accelerometer-2011-06-06-10-03-47-comb_hair-f1.txt\n",
      "Accelerometer-2011-06-06-10-05-40-comb_hair-f1.txt\n",
      "Accelerometer-2011-06-06-10-08-56-comb_hair-f1.txt\n",
      "Accelerometer-2011-06-06-10-10-30-comb_hair-f1.txt\n",
      "Accelerometer-2011-03-24-10-24-02-descend_stairs-f1.txt\n",
      "Accelerometer-2011-03-24-10-25-11-descend_stairs-f1.txt\n",
      "Accelerometer-2011-03-24-11-20-43-descend_stairs-f1.txt\n",
      "Accelerometer-2011-03-29-09-15-58-descend_stairs-f1.txt\n",
      "Accelerometer-2011-03-29-16-16-34-descend_stairs-f1.txt\n",
      "Accelerometer-2011-03-29-16-17-18-descend_stairs-f1.txt\n",
      "Accelerometer-2011-03-29-16-17-50-descend_stairs-f1.txt\n",
      "Accelerometer-2011-03-29-16-18-14-descend_stairs-f1.txt\n",
      "Accelerometer-2011-04-05-18-26-06-descend_stairs-f1.txt\n",
      "Accelerometer-2011-04-11-11-51-27-descend_stairs-f1.txt\n",
      "Accelerometer-2011-04-11-13-32-24-descend_stairs-f1.txt\n",
      "Accelerometer-2011-05-30-08-20-55-descend_stairs-f1.txt\n",
      "Accelerometer-2011-05-30-08-24-58-descend_stairs-f1.txt\n",
      "Accelerometer-2011-05-30-08-34-06-descend_stairs-f1.txt\n",
      "Accelerometer-2011-05-30-09-29-15-descend_stairs-f1.txt\n",
      "Accelerometer-2011-05-30-09-31-43-descend_stairs-f1.txt\n",
      "Accelerometer-2011-05-30-09-40-50-descend_stairs-f1.txt\n",
      "Accelerometer-2011-05-30-10-19-56-descend_stairs-m1.txt\n",
      "Accelerometer-2011-05-30-10-24-51-descend_stairs-m1.txt\n",
      "Accelerometer-2011-05-30-10-37-30-descend_stairs-m1.txt\n",
      "Accelerometer-2011-05-30-20-48-10-descend_stairs-f1.txt\n",
      "Accelerometer-2011-05-30-20-56-43-descend_stairs-f1.txt\n",
      "Accelerometer-2011-05-30-21-09-46-descend_stairs-f1.txt\n",
      "Accelerometer-2011-05-30-21-32-18-descend_stairs-m2.txt\n",
      "Accelerometer-2011-05-30-21-39-11-descend_stairs-m2.txt\n",
      "Accelerometer-2011-05-30-21-53-35-descend_stairs-m2.txt\n",
      "Accelerometer-2011-05-31-14-52-31-descend_stairs-f1.txt\n",
      "Accelerometer-2011-05-31-15-00-30-descend_stairs-f1.txt\n",
      "Accelerometer-2011-05-31-15-15-48-descend_stairs-f1.txt\n",
      "Accelerometer-2011-05-31-16-21-49-descend_stairs-f1.txt\n",
      "Accelerometer-2011-05-31-16-28-22-descend_stairs-f1.txt\n",
      "Accelerometer-2011-06-02-10-36-04-descend_stairs-f1.txt\n",
      "Accelerometer-2011-06-02-10-36-42-descend_stairs-f1.txt\n",
      "Accelerometer-2011-06-02-10-38-15-descend_stairs-f1.txt\n",
      "Accelerometer-2011-06-02-10-38-52-descend_stairs-f1.txt\n",
      "Accelerometer-2011-06-06-10-56-23-descend_stairs-f1.txt\n",
      "Accelerometer-2011-06-06-10-56-55-descend_stairs-f1.txt\n",
      "Accelerometer-2011-06-06-10-58-14-descend_stairs-f1.txt\n",
      "Accelerometer-2011-06-06-10-58-40-descend_stairs-f1.txt\n",
      "Accelerometer-2011-06-06-10-59-38-descend_stairs-f1.txt\n",
      "Accelerometer-2011-06-06-11-00-26-descend_stairs-f1.txt\n",
      "Accelerometer-2011-06-06-11-01-54-descend_stairs-f1.txt\n",
      "Accelerometer-2011-03-24-10-07-02-drink_glass-f1.txt\n",
      "Accelerometer-2011-03-24-10-16-02-drink_glass-f1.txt\n",
      "Accelerometer-2011-03-24-10-46-25-drink_glass-f1.txt\n",
      "Accelerometer-2011-03-24-11-14-00-drink_glass-f1.txt\n",
      "Accelerometer-2011-03-24-13-09-29-drink_glass-f1.txt\n",
      "Accelerometer-2011-03-24-13-17-06-drink_glass-f1.txt\n",
      "Accelerometer-2011-03-24-13-31-22-drink_glass-f1.txt\n",
      "Accelerometer-2011-03-24-16-08-10-drink_glass-f2.txt\n",
      "Accelerometer-2011-03-24-16-08-29-drink_glass-f2.txt\n",
      "Accelerometer-2011-04-05-18-55-33-drink_glass-f1.txt\n",
      "Accelerometer-2011-04-08-17-33-35-drink_glass-f3.txt\n",
      "Accelerometer-2011-04-08-17-35-00-drink_glass-f3.txt\n",
      "Accelerometer-2011-04-08-18-10-09-drink_glass-m4.txt\n",
      "Accelerometer-2011-04-11-12-55-39-drink_glass-f1.txt\n",
      "Accelerometer-2011-04-11-13-17-55-drink_glass-f1.txt\n",
      "Accelerometer-2011-04-12-21-43-48-drink_glass-m8.txt\n",
      "Accelerometer-2011-04-12-21-44-55-drink_glass-m8.txt\n",
      "Accelerometer-2011-05-30-21-06-15-drink_glass-f1.txt\n",
      "Accelerometer-2011-05-30-21-06-42-drink_glass-f1.txt\n",
      "Accelerometer-2011-05-30-21-07-10-drink_glass-f1.txt\n",
      "Accelerometer-2011-05-30-21-51-35-drink_glass-m2.txt\n",
      "Accelerometer-2011-05-30-21-51-59-drink_glass-m2.txt\n",
      "Accelerometer-2011-05-30-21-52-25-drink_glass-m2.txt\n",
      "Accelerometer-2011-05-31-15-12-05-drink_glass-f1.txt\n",
      "Accelerometer-2011-05-31-15-12-36-drink_glass-f1.txt\n",
      "Accelerometer-2011-05-31-15-13-09-drink_glass-f1.txt\n",
      "Accelerometer-2011-05-31-16-35-12-drink_glass-f1.txt\n",
      "Accelerometer-2011-05-31-16-35-41-drink_glass-f1.txt\n",
      "Accelerometer-2011-05-31-16-37-39-drink_glass-f1.txt\n",
      "Accelerometer-2011-06-01-14-13-57-drink_glass-f1.txt\n",
      "Accelerometer-2011-06-01-14-14-22-drink_glass-f1.txt\n",
      "Accelerometer-2011-06-01-14-14-49-drink_glass-f1.txt\n",
      "Accelerometer-2011-06-01-14-15-19-drink_glass-f1.txt\n",
      "Accelerometer-2011-06-01-14-15-49-drink_glass-f1.txt\n",
      "Accelerometer-2011-06-01-14-20-46-drink_glass-f1.txt\n",
      "Accelerometer-2011-06-01-14-21-52-drink_glass-f1.txt\n",
      "Accelerometer-2011-06-01-14-23-15-drink_glass-f1.txt\n",
      "Accelerometer-2011-06-01-14-24-51-drink_glass-f1.txt\n",
      "Accelerometer-2011-06-01-14-28-50-drink_glass-f1.txt\n",
      "Accelerometer-2011-06-01-14-30-43-drink_glass-f1.txt\n",
      "Accelerometer-2011-06-01-14-33-59-drink_glass-f1.txt\n",
      "Accelerometer-2011-06-01-15-04-40-drink_glass-f1.txt\n",
      "Accelerometer-2011-06-01-15-09-37-drink_glass-f1.txt\n",
      "Accelerometer-2011-06-01-15-14-42-drink_glass-f1.txt\n",
      "Accelerometer-2011-06-01-15-25-43-drink_glass-f1.txt\n",
      "Accelerometer-2011-06-01-16-44-44-drink_glass-f1.txt\n",
      "Accelerometer-2011-06-01-16-45-39-drink_glass-f1.txt\n",
      "Accelerometer-2011-06-01-16-47-29-drink_glass-f1.txt\n",
      "Accelerometer-2011-06-01-16-49-14-drink_glass-f1.txt\n",
      "Accelerometer-2011-06-01-16-53-24-drink_glass-f1.txt\n",
      "Accelerometer-2011-06-01-16-57-10-drink_glass-f1.txt\n",
      "Accelerometer-2011-06-02-16-51-25-drink_glass-f4.txt\n",
      "Accelerometer-2011-06-02-16-51-58-drink_glass-f4.txt\n",
      "Accelerometer-2011-06-02-16-52-53-drink_glass-f4.txt\n",
      "Accelerometer-2011-06-02-17-19-56-drink_glass-m1.txt\n",
      "Accelerometer-2011-06-02-17-20-17-drink_glass-m1.txt\n",
      "Accelerometer-2011-06-02-17-20-47-drink_glass-m1.txt\n",
      "Accelerometer-2011-06-02-17-23-53-drink_glass-m1.txt\n",
      "Accelerometer-2011-06-02-17-24-39-drink_glass-m1.txt\n",
      "Accelerometer-2011-06-02-17-29-13-drink_glass-m1.txt\n",
      "Accelerometer-2011-06-02-17-29-50-drink_glass-m1.txt\n",
      "Accelerometer-2011-06-02-17-30-51-drink_glass-m1.txt\n",
      "Accelerometer-2011-06-02-17-32-31-drink_glass-m1.txt\n",
      "Accelerometer-2011-06-02-17-40-40-drink_glass-m1.txt\n",
      "Accelerometer-2011-06-02-17-52-26-drink_glass-m1.txt\n",
      "Accelerometer-2012-03-23-03-47-02-drink_glass-m9.txt\n",
      "Accelerometer-2012-03-23-03-47-36-drink_glass-m9.txt\n",
      "Accelerometer-2012-03-23-03-54-54-drink_glass-m9.txt\n",
      "Accelerometer-2012-03-26-04-53-07-drink_glass-f2.txt\n",
      "Accelerometer-2012-03-26-04-56-11-drink_glass-f2.txt\n",
      "Accelerometer-2012-03-26-05-02-06-drink_glass-m3.txt\n",
      "Accelerometer-2012-03-26-05-03-53-drink_glass-m3.txt\n",
      "Accelerometer-2012-05-25-18-30-30-drink_glass-f4.txt\n",
      "Accelerometer-2012-05-25-18-31-18-drink_glass-f4.txt\n",
      "Accelerometer-2012-05-25-18-32-32-drink_glass-f4.txt\n",
      "Accelerometer-2012-05-25-18-33-57-drink_glass-f4.txt\n",
      "Accelerometer-2012-05-25-18-36-27-drink_glass-f4.txt\n",
      "Accelerometer-2012-05-28-17-42-29-drink_glass-m1.txt\n",
      "Accelerometer-2012-05-28-17-44-02-drink_glass-m1.txt\n",
      "Accelerometer-2012-05-28-17-44-31-drink_glass-m1.txt\n",
      "Accelerometer-2012-05-28-17-47-23-drink_glass-m1.txt\n",
      "Accelerometer-2012-05-28-17-47-49-drink_glass-m1.txt\n",
      "Accelerometer-2012-05-28-17-50-04-drink_glass-m1.txt\n",
      "Accelerometer-2012-05-29-16-42-17-drink_glass-f2.txt\n",
      "Accelerometer-2012-05-29-16-44-12-drink_glass-f2.txt\n",
      "Accelerometer-2012-05-29-16-48-07-drink_glass-f2.txt\n",
      "Accelerometer-2012-05-29-17-11-01-drink_glass-m3.txt\n",
      "Accelerometer-2012-05-29-17-20-28-drink_glass-m3.txt\n",
      "Accelerometer-2012-05-30-19-51-49-drink_glass-f2.txt\n",
      "Accelerometer-2012-05-30-19-53-19-drink_glass-f2.txt\n",
      "Accelerometer-2012-05-30-19-53-49-drink_glass-f2.txt\n",
      "Accelerometer-2012-05-30-19-54-48-drink_glass-f2.txt\n",
      "Accelerometer-2012-05-30-19-57-08-drink_glass-m3.txt\n",
      "Accelerometer-2012-05-30-19-58-10-drink_glass-m3.txt\n",
      "Accelerometer-2012-05-30-19-58-44-drink_glass-m3.txt\n",
      "Accelerometer-2012-05-30-19-59-40-drink_glass-m3.txt\n",
      "Accelerometer-2012-05-30-21-48-13-drink_glass-m2.txt\n",
      "Accelerometer-2012-05-30-21-48-45-drink_glass-m2.txt\n",
      "Accelerometer-2012-05-30-22-10-16-drink_glass-m2.txt\n",
      "Accelerometer-2012-05-30-22-10-40-drink_glass-m2.txt\n",
      "Accelerometer-2011-03-24-13-06-15-eat_meat-f1.txt\n",
      "Accelerometer-2011-03-24-13-10-14-eat_meat-f1.txt\n",
      "Accelerometer-2011-03-24-13-12-52-eat_meat-f1.txt\n",
      "Accelerometer-2011-03-24-13-17-42-eat_meat-f1.txt\n",
      "Accelerometer-2011-03-24-13-21-39-eat_meat-f1.txt\n",
      "Accelerometer-2011-03-24-13-33-22-eat_soup-f1.txt\n",
      "Accelerometer-2011-03-24-13-44-18-eat_soup-f1.txt\n",
      "Accelerometer-2011-03-24-13-56-42-eat_soup-f1.txt\n",
      "Accelerometer-2011-03-29-09-21-17-getup_bed-f1.txt\n",
      "Accelerometer-2011-03-29-09-24-50-getup_bed-f1.txt\n",
      "Accelerometer-2011-03-29-09-49-03-getup_bed-f1.txt\n",
      "Accelerometer-2011-04-05-18-28-21-getup_bed-f1.txt\n",
      "Accelerometer-2011-04-11-11-53-01-getup_bed-f1.txt\n",
      "Accelerometer-2011-04-11-13-34-05-getup_bed-f1.txt\n",
      "Accelerometer-2011-05-30-08-26-56-getup_bed-f1.txt\n",
      "Accelerometer-2011-05-30-08-38-15-getup_bed-f1.txt\n",
      "Accelerometer-2011-05-30-09-33-26-getup_bed-f1.txt\n",
      "Accelerometer-2011-05-30-09-42-46-getup_bed-f1.txt\n",
      "Accelerometer-2011-05-30-10-27-01-getup_bed-m1.txt\n",
      "Accelerometer-2011-05-30-10-39-27-getup_bed-m1.txt\n",
      "Accelerometer-2011-05-30-20-53-40-getup_bed-f1.txt\n",
      "Accelerometer-2011-05-30-20-59-57-getup_bed-f1.txt\n",
      "Accelerometer-2011-05-30-21-13-57-getup_bed-f1.txt\n",
      "Accelerometer-2011-05-30-21-36-01-getup_bed-m2.txt\n",
      "Accelerometer-2011-05-30-21-42-30-getup_bed-m2.txt\n",
      "Accelerometer-2011-05-30-21-59-46-getup_bed-m2.txt\n",
      "Accelerometer-2011-05-31-14-57-24-getup_bed-f1.txt\n",
      "Accelerometer-2011-05-31-15-02-55-getup_bed-f1.txt\n",
      "Accelerometer-2011-05-31-15-19-41-getup_bed-f1.txt\n",
      "Accelerometer-2011-05-31-16-25-09-getup_bed-f1.txt\n",
      "Accelerometer-2011-05-31-16-30-18-getup_bed-f1.txt\n",
      "Accelerometer-2011-06-02-16-57-57-getup_bed-f4.txt\n",
      "Accelerometer-2011-06-02-16-58-46-getup_bed-f4.txt\n",
      "Accelerometer-2011-06-02-17-00-08-getup_bed-f4.txt\n",
      "Accelerometer-2011-06-02-17-22-34-getup_bed-m1.txt\n",
      "Accelerometer-2012-05-28-17-52-54-getup_bed-m1.txt\n",
      "Accelerometer-2012-05-28-17-53-29-getup_bed-m1.txt\n",
      "Accelerometer-2012-05-28-17-54-04-getup_bed-m1.txt\n",
      "Accelerometer-2012-05-28-17-54-37-getup_bed-m1.txt\n",
      "Accelerometer-2012-05-28-17-55-04-getup_bed-m1.txt\n",
      "Accelerometer-2012-05-30-21-51-41-getup_bed-m2.txt\n",
      "Accelerometer-2012-05-30-21-52-18-getup_bed-m2.txt\n",
      "Accelerometer-2012-05-30-21-53-05-getup_bed-m2.txt\n",
      "Accelerometer-2012-05-30-21-53-38-getup_bed-m2.txt\n",
      "Accelerometer-2012-05-30-21-54-12-getup_bed-m2.txt\n",
      "Accelerometer-2012-06-07-22-15-16-getup_bed-f4.txt\n",
      "Accelerometer-2012-06-07-22-16-05-getup_bed-f4.txt\n",
      "Accelerometer-2012-06-07-22-16-42-getup_bed-f4.txt\n",
      "Accelerometer-2012-06-07-22-17-45-getup_bed-f4.txt\n",
      "Accelerometer-2012-06-07-22-19-25-getup_bed-f4.txt\n",
      "Accelerometer-2012-06-07-22-20-59-getup_bed-f4.txt\n",
      "Accelerometer-2012-06-07-22-23-46-getup_bed-f4.txt\n",
      "Accelerometer-2012-06-07-22-24-33-getup_bed-f4.txt\n",
      "Accelerometer-2012-06-07-22-26-32-getup_bed-f4.txt\n",
      "Accelerometer-2012-06-07-22-28-50-getup_bed-f4.txt\n",
      "Accelerometer-2012-06-07-22-30-31-getup_bed-f4.txt\n",
      "Accelerometer-2012-06-07-22-31-18-getup_bed-f4.txt\n",
      "Accelerometer-2012-06-12-15-28-34-getup_bed-m4.txt\n",
      "Accelerometer-2012-06-12-15-29-15-getup_bed-m4.txt\n",
      "Accelerometer-2012-06-12-15-30-07-getup_bed-m4.txt\n",
      "Accelerometer-2012-06-12-15-31-01-getup_bed-m4.txt\n",
      "Accelerometer-2012-06-12-15-31-56-getup_bed-m4.txt\n",
      "Accelerometer-2012-06-12-15-32-38-getup_bed-m4.txt\n",
      "Accelerometer-2012-06-12-15-34-01-getup_bed-m4.txt\n",
      "Accelerometer-2012-06-12-15-34-40-getup_bed-m4.txt\n",
      "Accelerometer-2012-06-12-15-35-29-getup_bed-m4.txt\n",
      "Accelerometer-2012-06-12-15-36-02-getup_bed-m4.txt\n",
      "Accelerometer-2012-06-12-15-50-48-getup_bed-f3.txt\n",
      "Accelerometer-2012-06-12-15-51-25-getup_bed-f3.txt\n",
      "Accelerometer-2012-06-12-15-52-04-getup_bed-f3.txt\n",
      "Accelerometer-2012-06-12-15-52-46-getup_bed-f3.txt\n",
      "Accelerometer-2012-06-12-15-53-20-getup_bed-f3.txt\n",
      "Accelerometer-2012-06-12-15-54-03-getup_bed-f3.txt\n",
      "Accelerometer-2012-06-12-15-54-33-getup_bed-f3.txt\n",
      "Accelerometer-2012-06-12-15-56-41-getup_bed-f3.txt\n",
      "Accelerometer-2012-06-12-15-57-21-getup_bed-f3.txt\n",
      "Accelerometer-2012-06-12-15-58-01-getup_bed-f3.txt\n",
      "Accelerometer-2012-06-12-18-39-37-getup_bed-f2.txt\n",
      "Accelerometer-2012-06-12-18-40-26-getup_bed-f2.txt\n",
      "Accelerometer-2012-06-12-18-41-40-getup_bed-f2.txt\n",
      "Accelerometer-2012-06-12-18-42-44-getup_bed-f2.txt\n",
      "Accelerometer-2012-06-12-18-43-26-getup_bed-f2.txt\n",
      "Accelerometer-2012-06-12-18-44-04-getup_bed-f2.txt\n",
      "Accelerometer-2012-06-12-18-44-53-getup_bed-f2.txt\n",
      "Accelerometer-2012-06-12-18-48-47-getup_bed-m3.txt\n",
      "Accelerometer-2012-06-12-18-49-31-getup_bed-m3.txt\n",
      "Accelerometer-2012-06-12-18-50-08-getup_bed-m3.txt\n",
      "Accelerometer-2012-06-12-18-50-50-getup_bed-m3.txt\n",
      "Accelerometer-2012-06-12-18-51-33-getup_bed-m3.txt\n",
      "Accelerometer-2012-06-12-18-52-06-getup_bed-m3.txt\n",
      "Accelerometer-2012-06-12-18-52-37-getup_bed-m3.txt\n",
      "Accelerometer-2012-06-20-20-27-19-getup_bed-f5.txt\n",
      "Accelerometer-2012-06-20-20-28-27-getup_bed-f5.txt\n",
      "Accelerometer-2012-06-20-20-29-11-getup_bed-f5.txt\n",
      "Accelerometer-2012-06-20-20-29-49-getup_bed-f5.txt\n",
      "Accelerometer-2012-06-20-20-30-45-getup_bed-f5.txt\n",
      "Accelerometer-2012-06-20-20-31-40-getup_bed-f5.txt\n",
      "Accelerometer-2012-06-20-20-32-40-getup_bed-f5.txt\n",
      "Accelerometer-2012-06-20-20-33-22-getup_bed-f5.txt\n",
      "Accelerometer-2012-06-20-20-34-30-getup_bed-f5.txt\n",
      "Accelerometer-2012-06-20-21-52-24-getup_bed-m10.txt\n",
      "Accelerometer-2012-06-20-21-53-06-getup_bed-m10.txt\n",
      "Accelerometer-2012-06-20-21-53-40-getup_bed-m10.txt\n",
      "Accelerometer-2012-06-20-21-55-30-getup_bed-m10.txt\n",
      "Accelerometer-2012-06-20-21-56-36-getup_bed-m10.txt\n",
      "Accelerometer-2012-06-20-21-57-33-getup_bed-m10.txt\n",
      "Accelerometer-2012-06-20-21-58-13-getup_bed-m10.txt\n",
      "Accelerometer-2012-06-20-21-59-57-getup_bed-m10.txt\n",
      "Accelerometer-2012-06-20-22-00-55-getup_bed-m10.txt\n",
      "Accelerometer-2011-03-29-09-19-22-liedown_bed-f1.txt\n",
      "Accelerometer-2011-03-29-09-23-22-liedown_bed-f1.txt\n",
      "Accelerometer-2011-03-29-09-45-46-liedown_bed-f1.txt\n",
      "Accelerometer-2011-04-05-18-27-12-liedown_bed-f1.txt\n",
      "Accelerometer-2011-04-11-11-52-20-liedown_bed-f1.txt\n",
      "Accelerometer-2011-04-11-13-33-26-liedown_bed-f1.txt\n",
      "Accelerometer-2011-05-30-08-25-59-liedown_bed-f1.txt\n",
      "Accelerometer-2011-05-30-08-37-27-liedown_bed-f1.txt\n",
      "Accelerometer-2011-05-30-09-32-42-liedown_bed-f1.txt\n",
      "Accelerometer-2011-05-30-09-41-49-liedown_bed-f1.txt\n",
      "Accelerometer-2011-05-30-10-26-14-liedown_bed-m1.txt\n",
      "Accelerometer-2011-05-30-10-38-41-liedown_bed-m1.txt\n",
      "Accelerometer-2011-05-30-20-52-31-liedown_bed-f1.txt\n",
      "Accelerometer-2011-05-30-20-59-04-liedown_bed-f1.txt\n",
      "Accelerometer-2011-05-30-21-13-15-liedown_bed-f1.txt\n",
      "Accelerometer-2011-05-30-21-35-19-liedown_bed-m2.txt\n",
      "Accelerometer-2011-05-30-21-41-32-liedown_bed-m2.txt\n",
      "Accelerometer-2011-05-30-21-59-16-liedown_bed-m2.txt\n",
      "Accelerometer-2011-05-31-14-56-04-liedown_bed-f1.txt\n",
      "Accelerometer-2011-05-31-15-02-19-liedown_bed-f1.txt\n",
      "Accelerometer-2011-05-31-15-18-50-liedown_bed-f1.txt\n",
      "Accelerometer-2011-05-31-16-24-36-liedown_bed-f1.txt\n",
      "Accelerometer-2011-05-31-16-29-43-liedown_bed-f1.txt\n",
      "Accelerometer-2011-06-02-16-57-35-liedown_bed-f4.txt\n",
      "Accelerometer-2011-06-02-16-58-20-liedown_bed-f4.txt\n",
      "Accelerometer-2011-06-02-16-59-13-liedown_bed-f4.txt\n",
      "Accelerometer-2011-06-02-17-00-38-liedown_bed-f4.txt\n",
      "Accelerometer-2011-06-02-17-21-57-liedown_bed-m1.txt\n",
      "Accelerometer-2011-03-24-10-04-32-pour_water-f1.txt\n",
      "Accelerometer-2011-03-24-10-51-12-pour_water-f1.txt\n",
      "Accelerometer-2011-03-24-13-30-01-pour_water-f1.txt\n",
      "Accelerometer-2011-04-05-18-54-14-pour_water-f1.txt\n",
      "Accelerometer-2011-04-11-12-55-26-pour_water-f1.txt\n",
      "Accelerometer-2011-05-30-21-05-39-pour_water-f1.txt\n",
      "Accelerometer-2011-05-30-21-51-05-pour_water-m2.txt\n",
      "Accelerometer-2011-05-31-15-11-25-pour_water-f1.txt\n",
      "Accelerometer-2011-05-31-16-34-32-pour_water-f1.txt\n",
      "Accelerometer-2011-06-01-14-12-53-pour_water-f1.txt\n",
      "Accelerometer-2011-06-01-14-13-29-pour_water-f1.txt\n",
      "Accelerometer-2011-06-01-14-16-35-pour_water-f1.txt\n",
      "Accelerometer-2011-06-01-14-17-02-pour_water-f1.txt\n",
      "Accelerometer-2011-06-01-14-18-36-pour_water-f1.txt\n",
      "Accelerometer-2011-06-01-14-25-16-pour_water-f1.txt\n",
      "Accelerometer-2011-06-01-14-27-40-pour_water-f1.txt\n",
      "Accelerometer-2011-06-01-14-32-38-pour_water-f1.txt\n",
      "Accelerometer-2011-06-01-16-46-50-pour_water-f1.txt\n",
      "Accelerometer-2011-06-02-16-50-40-pour_water-f4.txt\n",
      "Accelerometer-2011-06-02-16-52-30-pour_water-f4.txt\n",
      "Accelerometer-2011-06-02-17-04-09-pour_water-f4.txt\n",
      "Accelerometer-2011-06-02-17-06-15-pour_water-f4.txt\n",
      "Accelerometer-2011-06-02-17-06-44-pour_water-f4.txt\n",
      "Accelerometer-2011-06-02-17-23-16-pour_water-m1.txt\n",
      "Accelerometer-2011-06-02-17-28-27-pour_water-m1.txt\n",
      "Accelerometer-2011-06-02-17-33-30-pour_water-m1.txt\n",
      "Accelerometer-2011-06-02-17-45-45-pour_water-m1.txt\n",
      "Accelerometer-2011-06-06-09-39-30-pour_water-f1.txt\n",
      "Accelerometer-2011-06-06-09-40-34-pour_water-f1.txt\n",
      "Accelerometer-2011-06-06-09-42-11-pour_water-f1.txt\n",
      "Accelerometer-2011-06-06-09-44-12-pour_water-f1.txt\n",
      "Accelerometer-2011-06-06-09-45-45-pour_water-f1.txt\n",
      "Accelerometer-2011-06-06-09-46-20-pour_water-f1.txt\n",
      "Accelerometer-2011-06-06-09-47-48-pour_water-f1.txt\n",
      "Accelerometer-2011-06-06-09-48-59-pour_water-f1.txt\n",
      "Accelerometer-2011-06-06-09-49-51-pour_water-f1.txt\n",
      "Accelerometer-2011-06-06-09-50-17-pour_water-f1.txt\n",
      "Accelerometer-2011-06-06-09-51-53-pour_water-f1.txt\n",
      "Accelerometer-2011-06-06-09-55-25-pour_water-f1.txt\n",
      "Accelerometer-2011-06-06-09-56-28-pour_water-f1.txt\n",
      "Accelerometer-2012-03-23-03-46-21-pour_water-m9.txt\n",
      "Accelerometer-2012-03-23-03-54-26-pour_water-m9.txt\n",
      "Accelerometer-2012-03-23-03-55-46-pour_water-m9.txt\n",
      "Accelerometer-2012-03-26-04-53-59-pour_water-f2.txt\n",
      "Accelerometer-2012-03-26-04-55-40-pour_water-f2.txt\n",
      "Accelerometer-2012-03-26-05-01-39-pour_water-m3.txt\n",
      "Accelerometer-2012-03-26-05-03-27-pour_water-m3.txt\n",
      "Accelerometer-2012-05-25-18-29-48-pour_water-f4.txt\n",
      "Accelerometer-2012-05-25-18-31-58-pour_water-f4.txt\n",
      "Accelerometer-2012-05-25-18-34-46-pour_water-f4.txt\n",
      "Accelerometer-2012-05-25-18-37-06-pour_water-f4.txt\n",
      "Accelerometer-2012-05-28-17-41-55-pour_water-m1.txt\n",
      "Accelerometer-2012-05-28-17-43-20-pour_water-m1.txt\n",
      "Accelerometer-2012-05-28-17-46-43-pour_water-m1.txt\n",
      "Accelerometer-2012-05-28-17-49-34-pour_water-m1.txt\n",
      "Accelerometer-2012-05-29-16-43-41-pour_water-f2.txt\n",
      "Accelerometer-2012-05-29-16-49-00-pour_water-f2.txt\n",
      "Accelerometer-2012-05-29-17-10-30-pour_water-m3.txt\n",
      "Accelerometer-2012-05-29-17-12-07-pour_water-m3.txt\n",
      "Accelerometer-2012-05-29-17-19-55-pour_water-m3.txt\n",
      "Accelerometer-2012-05-30-18-26-02-pour_water-f3.txt\n",
      "Accelerometer-2012-05-30-18-26-49-pour_water-f3.txt\n",
      "Accelerometer-2012-05-30-18-27-15-pour_water-f3.txt\n",
      "Accelerometer-2012-05-30-18-27-41-pour_water-f3.txt\n",
      "Accelerometer-2012-05-30-19-07-14-pour_water-m4.txt\n",
      "Accelerometer-2012-05-30-19-07-43-pour_water-m4.txt\n",
      "Accelerometer-2012-05-30-19-08-20-pour_water-m4.txt\n",
      "Accelerometer-2012-05-30-19-08-50-pour_water-m4.txt\n",
      "Accelerometer-2012-05-30-19-50-42-pour_water-f2.txt\n",
      "Accelerometer-2012-05-30-19-52-29-pour_water-f2.txt\n",
      "Accelerometer-2012-05-30-19-54-24-pour_water-f2.txt\n",
      "Accelerometer-2012-05-30-19-56-40-pour_water-m3.txt\n",
      "Accelerometer-2012-05-30-19-57-40-pour_water-m3.txt\n",
      "Accelerometer-2012-05-30-19-59-17-pour_water-m3.txt\n",
      "Accelerometer-2012-06-07-21-27-46-pour_water-f4.txt\n",
      "Accelerometer-2012-06-07-21-28-23-pour_water-f4.txt\n",
      "Accelerometer-2012-06-07-21-28-48-pour_water-f4.txt\n",
      "Accelerometer-2012-06-07-21-29-09-pour_water-f4.txt\n",
      "Accelerometer-2012-06-07-21-29-48-pour_water-f4.txt\n",
      "Accelerometer-2012-06-07-21-30-16-pour_water-f4.txt\n",
      "Accelerometer-2012-06-07-21-30-37-pour_water-f4.txt\n",
      "Accelerometer-2012-06-07-21-31-05-pour_water-f4.txt\n",
      "Accelerometer-2012-06-07-21-31-24-pour_water-f4.txt\n",
      "Accelerometer-2012-06-07-21-31-48-pour_water-f4.txt\n",
      "Accelerometer-2012-06-07-21-32-12-pour_water-f4.txt\n",
      "Accelerometer-2012-06-07-21-32-37-pour_water-f4.txt\n",
      "Accelerometer-2012-06-07-21-33-21-pour_water-f4.txt\n",
      "Accelerometer-2012-06-07-21-33-47-pour_water-f4.txt\n",
      "Accelerometer-2012-06-07-21-34-22-pour_water-f4.txt\n",
      "Accelerometer-2012-06-07-21-34-47-pour_water-f4.txt\n",
      "Accelerometer-2012-06-07-21-35-28-pour_water-f4.txt\n",
      "Accelerometer-2012-06-07-21-35-50-pour_water-f4.txt\n",
      "Accelerometer-2012-06-09-22-54-40-pour_water-m11.txt\n",
      "Accelerometer-2012-06-09-22-55-08-pour_water-m11.txt\n",
      "Accelerometer-2012-06-09-22-55-42-pour_water-m11.txt\n",
      "Accelerometer-2012-06-09-22-56-17-pour_water-m11.txt\n",
      "Accelerometer-2012-06-09-22-56-53-pour_water-m11.txt\n",
      "Accelerometer-2012-06-09-22-57-12-pour_water-m11.txt\n",
      "Accelerometer-2012-06-09-22-58-08-pour_water-m11.txt\n",
      "Accelerometer-2012-06-09-22-58-28-pour_water-m11.txt\n",
      "Accelerometer-2011-03-24-09-50-16-sitdown_chair-f1.txt\n",
      "Accelerometer-2011-03-24-10-27-33-sitdown_chair-f1.txt\n",
      "Accelerometer-2011-03-24-16-09-29-sitdown_chair-f2.txt\n",
      "Accelerometer-2011-03-29-09-06-48-sitdown_chair-f1.txt\n",
      "Accelerometer-2011-03-29-09-12-27-sitdown_chair-f1.txt\n",
      "Accelerometer-2011-04-05-18-22-14-sitdown_chair-f1.txt\n",
      "Accelerometer-2011-04-05-18-47-38-sitdown_chair-f1.txt\n",
      "Accelerometer-2011-04-08-17-33-10-sitdown_chair-f3.txt\n",
      "Accelerometer-2011-04-08-17-34-35-sitdown_chair-f3.txt\n",
      "Accelerometer-2011-04-08-18-09-40-sitdown_chair-m4.txt\n",
      "Accelerometer-2011-04-11-11-45-28-sitdown_chair-f1.txt\n",
      "Accelerometer-2011-04-12-21-42-24-sitdown_chair-m8.txt\n",
      "Accelerometer-2011-04-12-21-43-15-sitdown_chair-m8.txt\n",
      "Accelerometer-2011-05-30-08-22-28-sitdown_chair-f1.txt\n",
      "Accelerometer-2011-05-30-08-31-30-sitdown_chair-f1.txt\n",
      "Accelerometer-2011-05-30-09-23-12-sitdown_chair-f1.txt\n",
      "Accelerometer-2011-05-30-09-24-48-sitdown_chair-f1.txt\n",
      "Accelerometer-2011-05-30-10-21-50-sitdown_chair-m1.txt\n",
      "Accelerometer-2011-05-30-10-31-12-sitdown_chair-m1.txt\n",
      "Accelerometer-2011-05-30-20-48-57-sitdown_chair-f1.txt\n",
      "Accelerometer-2011-05-30-20-49-32-sitdown_chair-f1.txt\n",
      "Accelerometer-2011-05-30-20-55-28-sitdown_chair-f1.txt\n",
      "Accelerometer-2011-05-30-21-04-13-sitdown_chair-f1.txt\n",
      "Accelerometer-2011-05-30-21-33-15-sitdown_chair-m2.txt\n",
      "Accelerometer-2011-05-30-21-37-52-sitdown_chair-m2.txt\n",
      "Accelerometer-2011-05-30-21-48-01-sitdown_chair-m2.txt\n",
      "Accelerometer-2011-05-31-14-53-37-sitdown_chair-f1.txt\n",
      "Accelerometer-2011-05-31-14-59-27-sitdown_chair-f1.txt\n",
      "Accelerometer-2011-05-31-15-07-32-sitdown_chair-f1.txt\n",
      "Accelerometer-2011-05-31-16-22-37-sitdown_chair-f1.txt\n",
      "Accelerometer-2011-05-31-16-27-00-sitdown_chair-f1.txt\n",
      "Accelerometer-2011-05-31-16-33-27-sitdown_chair-f1.txt\n",
      "Accelerometer-2011-06-01-14-37-35-sitdown_chair-f1.txt\n",
      "Accelerometer-2011-06-01-14-38-38-sitdown_chair-f1.txt\n",
      "Accelerometer-2011-06-01-14-39-48-sitdown_chair-f1.txt\n",
      "Accelerometer-2011-06-01-14-40-31-sitdown_chair-f1.txt\n",
      "Accelerometer-2011-06-01-14-41-42-sitdown_chair-f1.txt\n",
      "Accelerometer-2011-06-01-14-43-55-sitdown_chair-f1.txt\n",
      "Accelerometer-2011-06-01-14-45-42-sitdown_chair-f1.txt\n",
      "Accelerometer-2011-06-01-14-46-43-sitdown_chair-f1.txt\n",
      "Accelerometer-2011-06-01-14-50-18-sitdown_chair-f1.txt\n",
      "Accelerometer-2011-06-01-14-51-28-sitdown_chair-f1.txt\n",
      "Accelerometer-2011-06-02-16-15-55-sitdown_chair-f1.txt\n",
      "Accelerometer-2011-06-02-16-17-10-sitdown_chair-f1.txt\n",
      "Accelerometer-2011-06-02-16-21-01-sitdown_chair-f1.txt\n",
      "Accelerometer-2011-06-02-16-21-36-sitdown_chair-f1.txt\n",
      "Accelerometer-2011-06-02-16-43-56-sitdown_chair-f4.txt\n",
      "Accelerometer-2011-06-02-16-44-37-sitdown_chair-f4.txt\n",
      "Accelerometer-2011-06-02-16-45-19-sitdown_chair-f4.txt\n",
      "Accelerometer-2011-06-02-16-46-07-sitdown_chair-f4.txt\n",
      "Accelerometer-2011-06-02-16-46-43-sitdown_chair-f4.txt\n",
      "Accelerometer-2011-06-02-16-53-58-sitdown_chair-f4.txt\n",
      "Accelerometer-2011-06-02-16-54-34-sitdown_chair-f4.txt\n",
      "Accelerometer-2011-06-02-16-55-35-sitdown_chair-f4.txt\n",
      "Accelerometer-2011-06-02-17-04-59-sitdown_chair-f4.txt\n",
      "Accelerometer-2011-06-02-17-05-36-sitdown_chair-f4.txt\n",
      "Accelerometer-2011-06-02-17-25-34-sitdown_chair-m1.txt\n",
      "Accelerometer-2011-06-02-17-26-23-sitdown_chair-m1.txt\n",
      "Accelerometer-2011-06-02-17-27-18-sitdown_chair-m1.txt\n",
      "Accelerometer-2011-06-02-17-32-07-sitdown_chair-m1.txt\n",
      "Accelerometer-2011-06-02-17-41-43-sitdown_chair-m1.txt\n",
      "Accelerometer-2011-06-02-17-42-45-sitdown_chair-m1.txt\n",
      "Accelerometer-2011-06-02-17-44-20-sitdown_chair-m1.txt\n",
      "Accelerometer-2011-06-02-17-45-12-sitdown_chair-m1.txt\n",
      "Accelerometer-2011-06-06-09-34-12-sitdown_chair-f1.txt\n",
      "Accelerometer-2011-06-06-09-35-11-sitdown_chair-f1.txt\n",
      "Accelerometer-2011-06-06-09-35-43-sitdown_chair-f1.txt\n",
      "Accelerometer-2011-06-06-09-36-06-sitdown_chair-f1.txt\n",
      "Accelerometer-2011-12-05-09-39-34-sitdown_chair-f1.txt\n",
      "Accelerometer-2011-12-05-09-44-08-sitdown_chair-f1.txt\n",
      "Accelerometer-2011-12-05-09-46-34-sitdown_chair-f1.txt\n",
      "Accelerometer-2011-12-05-09-48-48-sitdown_chair-f1.txt\n",
      "Accelerometer-2011-12-05-09-50-54-sitdown_chair-f1.txt\n",
      "Accelerometer-2011-12-05-09-53-32-sitdown_chair-f1.txt\n",
      "Accelerometer-2011-12-05-09-58-23-sitdown_chair-f1.txt\n",
      "Accelerometer-2011-12-11-08-11-11-sitdown_chair-f4.txt\n",
      "Accelerometer-2011-12-11-08-12-14-sitdown_chair-f4.txt\n",
      "Accelerometer-2011-12-11-08-13-16-sitdown_chair-f4.txt\n",
      "Accelerometer-2011-12-11-08-14-09-sitdown_chair-f4.txt\n",
      "Accelerometer-2011-12-11-08-15-21-sitdown_chair-f4.txt\n",
      "Accelerometer-2011-12-11-08-16-20-sitdown_chair-f4.txt\n",
      "Accelerometer-2011-12-11-08-19-32-sitdown_chair-m2.txt\n",
      "Accelerometer-2011-12-11-08-20-24-sitdown_chair-m2.txt\n",
      "Accelerometer-2011-12-11-08-21-20-sitdown_chair-m2.txt\n",
      "Accelerometer-2011-12-11-08-22-36-sitdown_chair-m2.txt\n",
      "Accelerometer-2011-12-11-08-23-42-sitdown_chair-m2.txt\n",
      "Accelerometer-2011-12-11-08-24-32-sitdown_chair-m2.txt\n",
      "Accelerometer-2012-03-23-03-45-26-sitdown_chair-m9.txt\n",
      "Accelerometer-2012-03-23-03-48-40-sitdown_chair-m9.txt\n",
      "Accelerometer-2012-03-26-04-54-57-sitdown_chair-f2.txt\n",
      "Accelerometer-2012-03-26-04-58-33-sitdown_chair-f2.txt\n",
      "Accelerometer-2012-03-26-05-02-58-sitdown_chair-m3.txt\n",
      "Accelerometer-2012-03-26-05-04-48-sitdown_chair-m3.txt\n",
      "Accelerometer-2012-05-25-18-27-57-sitdown_chair-f4.txt\n",
      "Accelerometer-2012-05-25-18-28-46-sitdown_chair-f4.txt\n",
      "Accelerometer-2012-05-25-18-33-29-sitdown_chair-f4.txt\n",
      "Accelerometer-2012-05-25-18-36-06-sitdown_chair-f4.txt\n",
      "Accelerometer-2012-05-28-17-40-02-sitdown_chair-m1.txt\n",
      "Accelerometer-2012-05-28-17-41-12-sitdown_chair-m1.txt\n",
      "Accelerometer-2012-05-28-17-46-13-sitdown_chair-m1.txt\n",
      "Accelerometer-2011-03-23-10-39-18-standup_chair-f1.txt\n",
      "Accelerometer-2011-03-23-10-42-01-standup_chair-f1.txt\n",
      "Accelerometer-2011-03-23-10-45-51-standup_chair-f1.txt\n",
      "Accelerometer-2011-03-23-10-47-59-standup_chair-f1.txt\n",
      "Accelerometer-2011-03-24-09-49-36-standup_chair-f1.txt\n",
      "Accelerometer-2011-03-24-10-38-17-standup_chair-f1.txt\n",
      "Accelerometer-2011-03-24-11-19-29-standup_chair-f1.txt\n",
      "Accelerometer-2011-03-24-16-09-19-standup_chair-f2.txt\n",
      "Accelerometer-2011-03-29-09-04-40-standup_chair-f1.txt\n",
      "Accelerometer-2011-03-29-09-10-24-standup_chair-f1.txt\n",
      "Accelerometer-2011-04-05-18-25-12-standup_chair-f1.txt\n",
      "Accelerometer-2011-04-08-17-32-45-standup_chair-f3.txt\n",
      "Accelerometer-2011-04-08-17-34-10-standup_chair-f3.txt\n",
      "Accelerometer-2011-04-08-18-09-06-standup_chair-m4.txt\n",
      "Accelerometer-2011-04-11-11-48-09-standup_chair-f1.txt\n",
      "Accelerometer-2011-04-11-13-24-10-standup_chair-f1.txt\n",
      "Accelerometer-2011-04-12-21-42-53-standup_chair-m8.txt\n",
      "Accelerometer-2011-04-12-21-45-21-standup_chair-m8.txt\n",
      "Accelerometer-2011-05-30-08-24-19-standup_chair-f1.txt\n",
      "Accelerometer-2011-05-30-08-32-03-standup_chair-f1.txt\n",
      "Accelerometer-2011-05-30-09-24-04-standup_chair-f1.txt\n",
      "Accelerometer-2011-05-30-09-25-14-standup_chair-f1.txt\n",
      "Accelerometer-2011-05-30-10-23-55-standup_chair-m1.txt\n",
      "Accelerometer-2011-05-30-10-31-41-standup_chair-m1.txt\n",
      "Accelerometer-2011-05-30-20-50-04-standup_chair-f1.txt\n",
      "Accelerometer-2011-05-30-20-50-30-standup_chair-f1.txt\n",
      "Accelerometer-2011-05-30-20-56-05-standup_chair-f1.txt\n",
      "Accelerometer-2011-05-30-21-07-51-standup_chair-f1.txt\n",
      "Accelerometer-2011-05-30-21-33-37-standup_chair-m2.txt\n",
      "Accelerometer-2011-05-30-21-38-19-standup_chair-m2.txt\n",
      "Accelerometer-2011-05-30-21-52-55-standup_chair-m2.txt\n",
      "Accelerometer-2011-05-31-14-54-09-standup_chair-f1.txt\n",
      "Accelerometer-2011-05-31-14-59-53-standup_chair-f1.txt\n",
      "Accelerometer-2011-05-31-15-13-49-standup_chair-f1.txt\n",
      "Accelerometer-2011-05-31-16-23-03-standup_chair-f1.txt\n",
      "Accelerometer-2011-05-31-16-27-45-standup_chair-f1.txt\n",
      "Accelerometer-2011-05-31-16-38-19-standup_chair-f1.txt\n",
      "Accelerometer-2011-06-01-14-38-03-standup_chair-f1.txt\n",
      "Accelerometer-2011-06-01-14-39-38-standup_chair-f1.txt\n",
      "Accelerometer-2011-06-01-14-40-19-standup_chair-f1.txt\n",
      "Accelerometer-2011-06-01-14-41-32-standup_chair-f1.txt\n",
      "Accelerometer-2011-06-01-14-43-42-standup_chair-f1.txt\n",
      "Accelerometer-2011-06-01-14-45-33-standup_chair-f1.txt\n",
      "Accelerometer-2011-06-01-14-46-35-standup_chair-f1.txt\n",
      "Accelerometer-2011-06-01-14-50-10-standup_chair-f1.txt\n",
      "Accelerometer-2011-06-01-14-51-21-standup_chair-f1.txt\n",
      "Accelerometer-2011-06-01-14-52-02-standup_chair-f1.txt\n",
      "Accelerometer-2011-06-02-16-43-22-standup_chair-f4.txt\n",
      "Accelerometer-2011-06-02-16-44-18-standup_chair-f4.txt\n",
      "Accelerometer-2011-06-02-16-44-59-standup_chair-f4.txt\n",
      "Accelerometer-2011-06-02-16-45-48-standup_chair-f4.txt\n",
      "Accelerometer-2011-06-02-16-46-25-standup_chair-f4.txt\n",
      "Accelerometer-2011-06-02-16-53-39-standup_chair-f4.txt\n",
      "Accelerometer-2011-06-02-16-54-15-standup_chair-f4.txt\n",
      "Accelerometer-2011-06-02-16-55-12-standup_chair-f4.txt\n",
      "Accelerometer-2011-06-02-17-04-40-standup_chair-f4.txt\n",
      "Accelerometer-2011-06-02-17-05-17-standup_chair-f4.txt\n",
      "Accelerometer-2011-06-02-17-25-13-standup_chair-m1.txt\n",
      "Accelerometer-2011-06-02-17-26-02-standup_chair-m1.txt\n",
      "Accelerometer-2011-06-02-17-26-55-standup_chair-m1.txt\n",
      "Accelerometer-2011-06-02-17-31-39-standup_chair-m1.txt\n",
      "Accelerometer-2011-06-02-17-41-11-standup_chair-m1.txt\n",
      "Accelerometer-2011-06-02-17-42-24-standup_chair-m1.txt\n",
      "Accelerometer-2011-06-02-17-43-41-standup_chair-m1.txt\n",
      "Accelerometer-2011-06-02-17-44-52-standup_chair-m1.txt\n",
      "Accelerometer-2011-12-05-09-38-00-standup_chair-f1.txt\n",
      "Accelerometer-2011-12-05-09-40-57-standup_chair-f1.txt\n",
      "Accelerometer-2011-12-05-09-45-47-standup_chair-f1.txt\n",
      "Accelerometer-2011-12-05-09-48-17-standup_chair-f1.txt\n",
      "Accelerometer-2011-12-05-09-50-23-standup_chair-f1.txt\n",
      "Accelerometer-2011-12-05-09-52-58-standup_chair-f1.txt\n",
      "Accelerometer-2011-12-05-09-55-36-standup_chair-f1.txt\n",
      "Accelerometer-2011-12-11-08-09-59-standup_chair-f4.txt\n",
      "Accelerometer-2011-12-11-08-11-36-standup_chair-f4.txt\n",
      "Accelerometer-2011-12-11-08-12-46-standup_chair-f4.txt\n",
      "Accelerometer-2011-12-11-08-13-41-standup_chair-f4.txt\n",
      "Accelerometer-2011-12-11-08-14-55-standup_chair-f4.txt\n",
      "Accelerometer-2011-12-11-08-15-55-standup_chair-f4.txt\n",
      "Accelerometer-2011-12-11-08-19-06-standup_chair-m2.txt\n",
      "Accelerometer-2011-12-11-08-19-58-standup_chair-m2.txt\n",
      "Accelerometer-2011-12-11-08-20-56-standup_chair-m2.txt\n",
      "Accelerometer-2011-12-11-08-22-05-standup_chair-m2.txt\n",
      "Accelerometer-2011-12-11-08-23-15-standup_chair-m2.txt\n",
      "Accelerometer-2011-12-11-08-24-09-standup_chair-m2.txt\n",
      "Accelerometer-2012-03-23-03-44-39-standup_chair-m9.txt\n",
      "Accelerometer-2012-03-23-03-48-20-standup_chair-m9.txt\n",
      "Accelerometer-2012-03-26-04-54-30-standup_chair-f2.txt\n",
      "Accelerometer-2012-03-26-04-58-13-standup_chair-f2.txt\n",
      "Accelerometer-2012-03-26-05-02-40-standup_chair-m3.txt\n",
      "Accelerometer-2012-03-26-05-04-33-standup_chair-m3.txt\n",
      "Accelerometer-2012-05-25-18-27-29-standup_chair-f4.txt\n",
      "Accelerometer-2012-05-25-18-28-24-standup_chair-f4.txt\n",
      "Accelerometer-2012-05-25-18-33-08-standup_chair-f4.txt\n",
      "Accelerometer-2012-05-25-18-35-48-standup_chair-f4.txt\n",
      "Accelerometer-2012-05-28-17-39-31-standup_chair-m1.txt\n",
      "Accelerometer-2012-05-28-17-40-45-standup_chair-m1.txt\n",
      "Accelerometer-2012-05-28-17-45-43-standup_chair-m1.txt\n",
      "Accelerometer-2012-05-28-17-50-40-standup_chair-m1.txt\n",
      "Accelerometer-2012-05-29-16-43-05-standup_chair-f2.txt\n",
      "Accelerometer-2012-05-29-16-47-10-standup_chair-f2.txt\n",
      "Accelerometer-2012-05-29-17-12-51-standup_chair-m3.txt\n",
      "Accelerometer-2012-05-29-17-14-27-standup_chair-m3.txt\n",
      "Accelerometer-2011-03-29-09-30-56-use_telephone-f1.txt\n",
      "Accelerometer-2011-03-29-09-32-59-use_telephone-f1.txt\n",
      "Accelerometer-2011-03-29-09-52-41-use_telephone-f1.txt\n",
      "Accelerometer-2011-04-05-18-29-19-use_telephone-f1.txt\n",
      "Accelerometer-2011-04-11-11-53-34-use_telephone-f1.txt\n",
      "Accelerometer-2011-04-11-11-54-10-use_telephone-f1.txt\n",
      "Accelerometer-2011-05-30-08-27-44-use_telephone-f1.txt\n",
      "Accelerometer-2011-05-30-09-34-12-use_telephone-f1.txt\n",
      "Accelerometer-2011-05-30-10-28-13-use_telephone-m1.txt\n",
      "Accelerometer-2011-05-30-21-00-40-use_telephone-f1.txt\n",
      "Accelerometer-2011-05-30-21-43-16-use_telephone-m2.txt\n",
      "Accelerometer-2011-05-31-15-03-53-use_telephone-f1.txt\n",
      "Accelerometer-2011-05-31-16-31-11-use_telephone-f1.txt\n",
      "Accelerometer-2011-03-24-09-51-07-walk-f1.txt\n",
      "Accelerometer-2011-03-24-09-52-11-walk-f1.txt\n",
      "Accelerometer-2011-03-24-10-58-34-walk-f1.txt\n",
      "Accelerometer-2011-03-24-11-20-24-walk-f1.txt\n",
      "Accelerometer-2011-03-24-11-21-13-walk-f1.txt\n",
      "Accelerometer-2011-03-29-16-16-01-walk-f1.txt\n",
      "Accelerometer-2011-04-05-18-30-57-walk-f1.txt\n",
      "Accelerometer-2011-04-11-11-55-57-walk-f1.txt\n",
      "Accelerometer-2011-05-30-08-29-07-walk-f1.txt\n",
      "Accelerometer-2011-05-30-09-35-03-walk-f1.txt\n",
      "Accelerometer-2011-05-30-10-29-28-walk-m1.txt\n",
      "Accelerometer-2011-05-30-20-51-03-walk-f1.txt\n",
      "Accelerometer-2011-05-30-20-57-19-walk-f1.txt\n",
      "Accelerometer-2011-05-30-21-01-39-walk-f1.txt\n",
      "Accelerometer-2011-05-30-21-34-15-walk-m2.txt\n",
      "Accelerometer-2011-05-30-21-39-55-walk-m2.txt\n",
      "Accelerometer-2011-05-30-21-44-35-walk-m2.txt\n",
      "Accelerometer-2011-05-31-14-54-41-walk-f1.txt\n",
      "Accelerometer-2011-05-31-15-01-05-walk-f1.txt\n",
      "Accelerometer-2011-05-31-15-05-44-walk-f1.txt\n",
      "Accelerometer-2011-05-31-16-23-32-walk-f1.txt\n",
      "Accelerometer-2011-05-31-16-28-55-walk-f1.txt\n",
      "Accelerometer-2011-05-31-16-32-12-walk-f1.txt\n",
      "Accelerometer-2012-05-29-16-45-30-walk-f2.txt\n",
      "Accelerometer-2012-05-29-16-46-02-walk-f2.txt\n",
      "Accelerometer-2012-05-29-16-51-03-walk-f2.txt\n",
      "Accelerometer-2012-05-29-16-51-40-walk-f2.txt\n",
      "Accelerometer-2012-05-29-17-13-45-walk-m3.txt\n",
      "Accelerometer-2012-05-29-17-15-15-walk-m3.txt\n",
      "Accelerometer-2012-05-29-17-16-24-walk-m3.txt\n",
      "Accelerometer-2012-05-29-17-18-55-walk-m3.txt\n",
      "Accelerometer-2012-05-30-18-28-32-walk-f3.txt\n",
      "Accelerometer-2012-05-30-18-29-02-walk-f3.txt\n",
      "Accelerometer-2012-05-30-18-29-35-walk-f3.txt\n",
      "Accelerometer-2012-05-30-18-30-08-walk-f3.txt\n",
      "Accelerometer-2012-05-30-19-09-54-walk-m4.txt\n",
      "Accelerometer-2012-05-30-19-10-27-walk-m4.txt\n",
      "Accelerometer-2012-05-30-19-10-56-walk-m4.txt\n",
      "Accelerometer-2012-05-30-19-11-27-walk-m4.txt\n",
      "Accelerometer-2012-05-30-22-02-42-walk-m2.txt\n",
      "Accelerometer-2012-05-30-22-03-03-walk-m2.txt\n",
      "Accelerometer-2012-05-30-22-03-27-walk-m2.txt\n",
      "Accelerometer-2012-05-30-22-03-52-walk-m2.txt\n",
      "Accelerometer-2012-05-30-22-04-20-walk-m2.txt\n",
      "Accelerometer-2012-06-06-08-56-33-walk-m5.txt\n",
      "Accelerometer-2012-06-06-08-57-26-walk-m5.txt\n",
      "Accelerometer-2012-06-06-08-58-20-walk-m5.txt\n",
      "Accelerometer-2012-06-06-08-59-00-walk-m5.txt\n",
      "Accelerometer-2012-06-06-09-02-06-walk-m5.txt\n",
      "Accelerometer-2012-06-06-09-02-54-walk-m5.txt\n",
      "Accelerometer-2012-06-06-09-03-57-walk-m5.txt\n",
      "Accelerometer-2012-06-06-09-04-41-walk-m5.txt\n",
      "Accelerometer-2012-06-06-09-07-30-walk-m5.txt\n",
      "Accelerometer-2012-06-06-09-08-19-walk-m5.txt\n",
      "Accelerometer-2012-06-06-09-09-21-walk-m5.txt\n",
      "Accelerometer-2012-06-06-09-10-02-walk-m5.txt\n",
      "Accelerometer-2012-06-06-09-34-20-walk-m6.txt\n",
      "Accelerometer-2012-06-06-09-35-07-walk-m6.txt\n",
      "Accelerometer-2012-06-06-09-36-06-walk-m6.txt\n",
      "Accelerometer-2012-06-06-09-36-38-walk-m6.txt\n",
      "Accelerometer-2012-06-06-09-39-47-walk-m6.txt\n",
      "Accelerometer-2012-06-06-09-40-33-walk-m6.txt\n",
      "Accelerometer-2012-06-06-09-41-32-walk-m6.txt\n",
      "Accelerometer-2012-06-06-09-42-05-walk-m6.txt\n",
      "Accelerometer-2012-06-06-09-45-03-walk-m6.txt\n",
      "Accelerometer-2012-06-06-09-45-38-walk-m6.txt\n",
      "Accelerometer-2012-06-06-09-46-31-walk-m6.txt\n",
      "Accelerometer-2012-06-06-09-47-07-walk-m6.txt\n",
      "Accelerometer-2012-06-06-14-06-10-walk-m7.txt\n",
      "Accelerometer-2012-06-06-14-06-43-walk-m7.txt\n",
      "Accelerometer-2012-06-06-14-07-41-walk-m7.txt\n",
      "Accelerometer-2012-06-06-14-08-12-walk-m7.txt\n",
      "Accelerometer-2012-06-06-14-10-02-walk-m7.txt\n",
      "Accelerometer-2012-06-06-14-10-33-walk-m7.txt\n",
      "Accelerometer-2012-06-06-14-11-30-walk-m7.txt\n",
      "Accelerometer-2012-06-06-14-12-00-walk-m7.txt\n",
      "Accelerometer-2012-06-06-14-13-56-walk-m7.txt\n",
      "Accelerometer-2012-06-06-14-14-27-walk-m7.txt\n",
      "Accelerometer-2012-06-06-14-15-25-walk-m7.txt\n",
      "Accelerometer-2012-06-06-14-15-57-walk-m7.txt\n",
      "Accelerometer-2012-06-07-10-50-09-walk-f1.txt\n",
      "Accelerometer-2012-06-07-10-50-42-walk-f1.txt\n",
      "Accelerometer-2012-06-07-10-52-14-walk-f1.txt\n",
      "Accelerometer-2012-06-07-10-52-46-walk-f1.txt\n",
      "Accelerometer-2012-06-07-10-55-33-walk-f1.txt\n",
      "Accelerometer-2012-06-07-10-57-12-walk-f1.txt\n",
      "Accelerometer-2012-06-07-10-57-45-walk-f1.txt\n",
      "Accelerometer-2012-06-11-11-23-05-walk-m1.txt\n",
      "Accelerometer-2012-06-11-11-30-57-walk-m1.txt\n",
      "Accelerometer-2012-06-11-11-31-42-walk-m1.txt\n",
      "Accelerometer-2012-06-11-11-32-28-walk-m1.txt\n",
      "Accelerometer-2012-06-11-11-33-07-walk-m1.txt\n",
      "Accelerometer-2012-06-11-11-33-42-walk-m1.txt\n",
      "Accelerometer-2012-06-11-11-34-18-walk-m1.txt\n",
      "Accelerometer-2012-06-11-11-36-25-walk-m1.txt\n",
      "Accelerometer-2012-06-11-11-37-01-walk-m1.txt\n",
      "Accelerometer-2012-06-11-11-37-36-walk-m1.txt\n",
      "Accelerometer-2012-06-11-11-38-12-walk-m1.txt\n",
      "Accelerometer-2012-06-11-11-38-52-walk-m1.txt\n",
      "Accelerometer-2012-06-11-11-39-29-walk-m1.txt\n"
     ]
    }
   ],
   "source": [
    "d = 'HMP_Dataset/'\n",
    "\n",
    "# filter list for all folders containing data (folders that don't start with .)\n",
    "file_list_filtered = [s for s in os.listdir(d)\n",
    "                      if os.path.isdir(os.path.join(d, s)) & ~fnmatch.fnmatch(s, '.*')]\n",
    "\n",
    "# create pandas data frame for all the data\n",
    "\n",
    "df = None\n",
    "\n",
    "for category in file_list_filtered:\n",
    "    data_files = os.listdir('HMP_Dataset/' + category)\n",
    "\n",
    "    # create a temporary pandas data frame for each data file\n",
    "    for data_file in data_files:\n",
    "        if sample < 1.0:\n",
    "            if random.random() > sample:\n",
    "                print('Skipping: ' + data_file)\n",
    "                continue\n",
    "        print(data_file)\n",
    "        temp_df = spark.read. \\\n",
    "            option(\"header\", \"false\"). \\\n",
    "            option(\"delimiter\", \" \"). \\\n",
    "            csv('HMP_Dataset/' + category + '/' + data_file, schema=schema)\n",
    "\n",
    "        # create a column called \"source\" storing the current CSV file\n",
    "        temp_df = temp_df.withColumn(\"source\", lit(data_file))\n",
    "\n",
    "        # create a column called \"class\" storing the current data folder\n",
    "        temp_df = temp_df.withColumn(\"class\", lit(category))\n",
    "\n",
    "        if df is None:\n",
    "            df = temp_df\n",
    "        else:\n",
    "            df = df.union(temp_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a4e0c1f4-f35c-4354-95ba-140584af27fa",
   "metadata": {},
   "source": [
    "Write dataframe to file in CSV format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "393e313f-990a-4266-a244-26fbd1b5c680",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "if Path(data_dir + data_csv).exists():\n",
    "    shutil.rmtree(data_dir + data_csv)\n",
    "    \n",
    "df.write.option(\"header\", \"true\").csv(data_dir + data_csv)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7169ba91-b8c9-41f3-923a-7d9c7919161b",
   "metadata": {
    "papermill": {
     "duration": 0.005346,
     "end_time": "2021-03-17T17:21:55.111968",
     "exception": false,
     "start_time": "2021-03-17T17:21:55.106622",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Convert CSV files to Parquet using Spark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "4e2127fb-1bba-4671-b6ae-1af35cfcef69",
   "metadata": {},
   "outputs": [],
   "source": [
    "parameters = list(\n",
    "    map(lambda s: re.sub('$', '\"', s),\n",
    "        map(\n",
    "            lambda s: s.replace('=', '=\"'),\n",
    "            filter(\n",
    "                lambda s: s.find('=') > -1 and bool(re.match(r'[A-Za-z0-9_]*=[.\\/A-Za-z0-9]*', s)),\n",
    "                sys.argv\n",
    "            )\n",
    "    )))\n",
    "\n",
    "for parameter in parameters:\n",
    "    logging.warning('Parameter: ' + parameter)\n",
    "    exec(parameter)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "2b68c484-d96a-4f0f-b90d-3ac3831d2440",
   "metadata": {},
   "outputs": [],
   "source": [
    "skip = False\n",
    "\n",
    "if os.path.exists(data_dir + output_data_parquet):\n",
    "    skip = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "3fa19d6e-7529-4188-a27c-9e8abca5996b",
   "metadata": {},
   "outputs": [],
   "source": [
    "if not skip:\n",
    "    df = spark.read.option('header', 'true').csv(data_dir + data_csv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "417f6daf-9071-438f-a01a-a74b77f34718",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "23/03/14 05:45:05 WARN hadoop.MemoryManager: Total allocation exceeds 95.00% (983,695,348 bytes) of heap memory\n",
      "Scaling row group sizes to 91.61% for 8 writers\n",
      "23/03/14 05:45:19 WARN hadoop.MemoryManager: Total allocation exceeds 95.00% (983,695,348 bytes) of heap memory\n",
      "Scaling row group sizes to 91.61% for 8 writers\n",
      "23/03/14 05:45:21 WARN hadoop.MemoryManager: Total allocation exceeds 95.00% (983,695,348 bytes) of heap memory\n",
      "Scaling row group sizes to 91.61% for 8 writers\n"
     ]
    }
   ],
   "source": [
    "if not skip:\n",
    "    df.write.parquet(data_dir + output_data_parquet)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6b456c27-f087-4c8d-940e-e38ab222449c",
   "metadata": {
    "papermill": {
     "duration": 0.005346,
     "end_time": "2021-03-17T17:21:55.111968",
     "exception": false,
     "start_time": "2021-03-17T17:21:55.106622",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Condense Parquet files using Spark"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eff30260-ded9-4915-b34e-e6419ee81397",
   "metadata": {},
   "source": [
    "Condenses a partitioned folder containing PARQUET files into a single PARQUET file by re-partitioning it to\n",
    "one and extracting the single PARQUET file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "744088f7-f4e9-4ba0-8576-eb1229616004",
   "metadata": {},
   "outputs": [],
   "source": [
    "# override parameters received from a potential call using %run magic\n",
    "parameters = list(\n",
    "    map(\n",
    "        lambda s: re.sub('$', '\"', s),\n",
    "        map(\n",
    "            lambda s: s.replace('=', '=\"'),\n",
    "            filter(\n",
    "                lambda s: s.find('=') > -1,\n",
    "                sys.argv\n",
    "            )\n",
    "        )\n",
    "    )\n",
    ")\n",
    "\n",
    "for parameter in parameters:\n",
    "    exec(parameter)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "8aa95994-5b11-4599-afa7-e162c1ed298c",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "df = spark.read.parquet(data_dir + data_parquet)\n",
    "\n",
    "df = df.repartition(1)\n",
    "\n",
    "df.write.parquet(data_dir + data_parquet_condensed)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "537cc791-4ab3-4854-a88e-f96040ec903e",
   "metadata": {},
   "source": [
    "Move the condensed Parquet file outside of the .parquet folder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "e0c5e643-b3b6-455f-960c-c5d3527b605c",
   "metadata": {},
   "outputs": [],
   "source": [
    "!mv {data_dir}/{data_parquet_condensed}/`ls {data_dir}/{data_parquet_condensed} |grep .parquet` {data_dir}/{data_parquet_condensed}_tmp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "79dba1b7-4bd4-4941-83e2-b0b0ef073e77",
   "metadata": {},
   "outputs": [],
   "source": [
    "!rm -Rf  {data_dir}/{data_parquet_condensed}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "8ce17025-3e08-4619-a42f-4b2647e97fae",
   "metadata": {},
   "outputs": [],
   "source": [
    "!mv {data_dir}/{data_parquet_condensed}_tmp {data_dir}/{data_parquet_condensed}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3fd1b3fa-ed69-4f67-88a6-4e6cba9cac24",
   "metadata": {
    "papermill": {
     "duration": 0.005346,
     "end_time": "2021-03-17T17:21:55.111968",
     "exception": false,
     "start_time": "2021-03-17T17:21:55.106622",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Convert condensed parquet file to CSV file with header using Spark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "75d90a0e-c106-483c-8d73-48344db5883e",
   "metadata": {},
   "outputs": [],
   "source": [
    "skip = False\n",
    "if os.path.exists(data_dir + data_csv_condensed):\n",
    "    skip = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "ec41bdc7-efc6-4741-a074-f41de95793f0",
   "metadata": {},
   "outputs": [],
   "source": [
    "if not skip:\n",
    "    df = spark.read.parquet(data_dir + data_parquet_condensed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "3fd5f3a6-cbeb-47b6-b34b-2917b5e75f6b",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "if not skip:\n",
    "    if os.path.exists(data_dir + data_csv_condensed):\n",
    "        shutil.rmtree(data_dir + data_csv_condensed)\n",
    "    df.coalesce(1).write.option(\"header\", \"true\").csv(data_dir + data_csv_condensed)\n",
    "    file = glob.glob(data_dir + data_csv_condensed + '/part-*')\n",
    "    shutil.move(file[0], data_dir + data_csv_condensed + '.tmp')\n",
    "    shutil.rmtree(data_dir + data_csv_condensed)\n",
    "    shutil.move(data_dir + data_csv_condensed + '.tmp', data_dir + data_csv_condensed)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "983a5ea9-8eca-4070-b674-9b5e987e69a1",
   "metadata": {
    "papermill": {
     "duration": 0.005346,
     "end_time": "2021-03-17T17:21:55.111968",
     "exception": false,
     "start_time": "2021-03-17T17:21:55.106622",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Spark Train Random Forest Classifier Model"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cedc0ed5-6c77-4204-814c-45c139bbd626",
   "metadata": {},
   "source": [
    "Train Random Forest Classifier Model with Apache SparkML"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "9234ffe5-004c-4552-aafc-1ed026b6b611",
   "metadata": {},
   "outputs": [],
   "source": [
    "parameters = list(\n",
    "    map(lambda s: re.sub('$', '\"', s),\n",
    "        map(\n",
    "            lambda s: s.replace('=', '=\"'),\n",
    "            filter(\n",
    "                lambda s: s.find('=') > -1 and bool(re.match(r'[A-Za-z0-9_]*=[.\\/A-Za-z0-9]*', s)),\n",
    "                sys.argv\n",
    "            )\n",
    "    )))\n",
    "\n",
    "for parameter in parameters:\n",
    "    logging.warning('Parameter: ' + parameter)\n",
    "    exec(parameter)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "854217c4-d5fc-481a-816a-ce54a2164bfb",
   "metadata": {},
   "outputs": [],
   "source": [
    "conf = SparkConf().setMaster(master)\n",
    "#if sys.version[0:3] == '3.6' or sys.version[0:3] == '3.7':\n",
    "conf.set(\"spark.jars\", 'jpmml-sparkml-executable-1.5.12.jar')\n",
    "\n",
    "sc = SparkContext.getOrCreate(conf)\n",
    "sqlContext = SQLContext(sc)\n",
    "spark = sqlContext.sparkSession"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "95bf31c1-7367-4cc8-a408-9f3cd169f8da",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Row(x='33', y='36', z='51', source='Accelerometer-2011-03-24-13-21-39-eat_meat-f1.txt', class='Eat_meat'),\n",
       " Row(x='33', y='36', z='51', source='Accelerometer-2011-03-24-13-21-39-eat_meat-f1.txt', class='Eat_meat'),\n",
       " Row(x='33', y='35', z='53', source='Accelerometer-2011-03-24-13-21-39-eat_meat-f1.txt', class='Eat_meat'),\n",
       " Row(x='31', y='37', z='52', source='Accelerometer-2011-03-24-13-21-39-eat_meat-f1.txt', class='Eat_meat'),\n",
       " Row(x='32', y='36', z='52', source='Accelerometer-2011-03-24-13-21-39-eat_meat-f1.txt', class='Eat_meat')]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = spark.read.format(\"csv\").option(\"header\", \"true\").load(os.path.join(data_dir, data_csv_condensed))\n",
    "df.limit(5).collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "d313c0c0-6138-4589-980b-83bbe6d19e76",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.sql.types import DoubleType\n",
    "df = df.withColumn(\"x\", df[\"x\"].cast(\"double\"))\n",
    "df = df.withColumn(\"y\", df[\"y\"].cast(\"double\"))\n",
    "df = df.withColumn(\"z\", df[\"z\"].cast(\"double\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "facca0ab-5291-4cb9-87cf-45c0aef6cd5e",
   "metadata": {},
   "outputs": [],
   "source": [
    "splits = df.randomSplit([0.8, 0.2])\n",
    "df_train = splits[0]\n",
    "df_test = splits[1]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0794502a-2be1-41fa-8df0-dd71c029f9c6",
   "metadata": {},
   "source": [
    "Use different hyperparameters and report the best performing hyperparameter combinations"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "24b04f14-c178-4faf-9e59-5ef424eaa0a4",
   "metadata": {},
   "source": [
    "Setting up the indexer, vector assembler and normalizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "510e2f10-d890-4095-8bd4-9e61ab1268f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "indexer = StringIndexer(inputCol=\"class\", outputCol=\"label\")\n",
    "\n",
    "vectorAssembler = VectorAssembler(inputCols=eval('[\"x\", \"y\", \"z\"]'), outputCol=\"features\")\n",
    "\n",
    "normalizer = MinMaxScaler(inputCol=\"features\", outputCol=\"features_norm\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4a51e41c-a6b2-4f97-b7f7-a6950e45fd5b",
   "metadata": {},
   "source": [
    "Check if index is correct"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "32b85f72-e97c-41da-933c-4b615a4e02ba",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[Row(class='Walk', label=0.0),\n",
       " Row(class='Getup_bed', label=1.0),\n",
       " Row(class='Drink_glass', label=2.0),\n",
       " Row(class='Pour_water', label=3.0),\n",
       " Row(class='Climb_stairs', label=4.0),\n",
       " Row(class='Eat_meat', label=5.0),\n",
       " Row(class='Brush_teeth', label=6.0),\n",
       " Row(class='Standup_chair', label=7.0),\n",
       " Row(class='Sitdown_chair', label=8.0),\n",
       " Row(class='Comb_hair', label=9.0),\n",
       " Row(class='Descend_stairs', label=10.0),\n",
       " Row(class='Use_telephone', label=11.0),\n",
       " Row(class='Liedown_bed', label=12.0),\n",
       " Row(class='Eat_soup', label=13.0)]"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipeline = Pipeline(stages=[indexer])\n",
    "\n",
    "df_ind = pipeline.fit(df_train).transform(df_train)\n",
    "df_ind.limit(5).collect()\n",
    "df_ind.sort('label').select('class','label').distinct().collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ba5253b7-d766-42a0-b674-3c52b8c12e35",
   "metadata": {},
   "source": [
    "Setting up evaluator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "047146a9-ce61-4cc7-a9ad-0e0cd08242b8",
   "metadata": {},
   "outputs": [],
   "source": [
    "binEval = MulticlassClassificationEvaluator(). \\\n",
    "    setMetricName(\"accuracy\"). \\\n",
    "    setPredictionCol(\"prediction\"). \\\n",
    "    setLabelCol(\"label\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "6eb93489-6146-46d3-be22-81735042e6b0",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "iterations = []\n",
    "'''\n",
    "for n_trees in treeCounts:\n",
    "    for depth in maxDepth:\n",
    "        rf = RandomForestClassifier(labelCol=\"label\", numTrees=n_trees, maxDepth=depth, predictionCol=\"prediction\")\n",
    "        \n",
    "        pipeling = Pipeline(stages=[indexer, vectorAssembler, normalizer, rf])\n",
    "        \n",
    "        model = pipeline.fit(df_train)\n",
    "        prediction = model.transform(df_test)\n",
    "        accuracy = binEval.evaluate(prediction)\n",
    "        iterations.append((n_trees, depth, accuracy))\n",
    "        \n",
    "'''\n",
    "\n",
    "for curNTrees in [10, 20]:\n",
    "    for curMaxDepth in [5, 7]:\n",
    "        rf = RandomForestClassifier(numTrees=curNTrees, maxDepth=curMaxDepth, seed=1)\n",
    "        \n",
    "        pipeline = Pipeline(stages=[indexer, vectorAssembler, normalizer, rf])\n",
    "\n",
    "        model = pipeline.fit(df_train)\n",
    "\n",
    "        prediction = model.transform(df_train)\n",
    "\n",
    "        binEval = MulticlassClassificationEvaluator().setMetricName(\"accuracy\").setPredictionCol(\"prediction\").setLabelCol(\"label\")\n",
    "\n",
    "        curAccuracy = binEval.evaluate(prediction)\n",
    "        \n",
    "        iterations.append((curNTrees, curMaxDepth, curAccuracy))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "13e4a5a2-973c-47eb-96ab-e84cfb084718",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(list(iterations))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "11a71f5c-fd53-49dd-b6f8-2b9187ca0436",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Number of Trees</th>\n",
       "      <th>Maximum Depth</th>\n",
       "      <th>Accuracy</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Combination</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>10</td>\n",
       "      <td>5</td>\n",
       "      <td>0.439178</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10</td>\n",
       "      <td>7</td>\n",
       "      <td>0.464036</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>20</td>\n",
       "      <td>5</td>\n",
       "      <td>0.443680</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>20</td>\n",
       "      <td>7</td>\n",
       "      <td>0.466432</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             Number of Trees  Maximum Depth  Accuracy\n",
       "Combination                                          \n",
       "0                         10              5  0.439178\n",
       "1                         10              7  0.464036\n",
       "2                         20              5  0.443680\n",
       "3                         20              7  0.466432"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.rename(columns={0: 'Number of Trees', 1: 'Maximum Depth', 2:'Accuracy'}, inplace=True)\n",
    "df.index.names= ['Combination']\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "03d23118-f4d8-4703-9922-a317781e0b95",
   "metadata": {},
   "source": [
    "Parameters producing the maximum accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "b651c4e1-9aef-4bcb-8ecc-c39fce0f9b9d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Number of Trees    20.000000\n",
       "Maximum Depth       7.000000\n",
       "Accuracy            0.466432\n",
       "Name: 3, dtype: float64"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.loc[df['Accuracy'].idxmax()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5f3872f8-4cc2-4436-9bf3-7d5201c030bd",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python",
   "language": "python",
   "name": "conda-env-python-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}