notebook.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "toc-hr-collapsed": false
   },
   "source": [
    "<h1>Splitting Video Section</h1>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "You need to provide the following:\n",
    "<ol>\n",
    "    <li>BUCKET_NAME</li>\n",
    "    <li>VIDEO_NAME</li>\n",
    "</ol>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import boto3, os\n",
    "s3 = boto3.client('s3')\n",
    "\n",
    "#S3 Bucket directory and video name\n",
    "BUCKET_NAME = \"csm.calpolydxhub\" #INPUT S3 BUCKET NAME\n",
    "VIDEO_NAME = \"scooter_Footage_pico.mp4\"#INPUT VIDEO NAME\n",
    "VIDEO_DIR = \"video/\"+VIDEO_NAME"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Download the video to this directory\n",
    "video = s3.download_file(BUCKET_NAME, VIDEO_DIR, VIDEO_NAME)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Make folder \"images\" in local jupyter notebook\n",
    "try:\n",
    "    os.mkdir(\"images\")\n",
    "except OSError:\n",
    "    print (\"Creation of the directory failed\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<br/>\n",
    "<h5>Split The Images From The Video</h5>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Split the video into <u>1 Image</u> per second here. <br/>\n",
    "Images will be stored localy in the local jupyter environment."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import cv2\n",
    "import math\n",
    "\n",
    "cap = cv2.VideoCapture(VIDEO_NAME)\n",
    "# frameRate = cap.get(5) #get frame rate of video\n",
    "frameRate = 30 #get frame rate of video\n",
    "Frames_Per_Second = .5\n",
    "print(\"FrameRate: \",frameRate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x=1\n",
    "while(cap.isOpened()):\n",
    "    frameId = cap.get(1) #current frame number\n",
    "    ret, frame = cap.read()\n",
    "    if (ret != True):\n",
    "        break\n",
    "    if (frameId % math.floor(frameRate//Frames_Per_Second) == 0):\n",
    "        filename = \"./images/\" +  '{0:05d}'.format(int(x)) + \".jpg\";\n",
    "        x+=1\n",
    "        cv2.imwrite(filename, frame)\n",
    "print(x, \" images\")\n",
    "cap.release()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h5>Upload Images To S3</h5>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Folder name to create and upload the images to in S3\n",
    "s3_imageFolder = \"images\" #INPUT S3 FOLDER NAME"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Maximum quantity of images that each sub-folder can contain\n",
    "images_per_folder = 1000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#The following function is used to upload directory to s3 via boto.\n",
    "def uploadDirectory(path,bucketname,s3_imageFolder):\n",
    "    image_count = 0\n",
    "    for root,dirs,files in os.walk(path):\n",
    "        for file in sorted(files):\n",
    "            images_subFolder = str(image_count//images_per_folder)\n",
    "            image_count +=1\n",
    "            s3.upload_file(os.path.join(root,file),bucketname, '{}/{}/{}'.format(s3_imageFolder,images_subFolder,file))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Upload the images to s3\n",
    "uploadDirectory(\"images\",BUCKET_NAME,s3_imageFolder)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h5>Clear Video And Images</h5>\n",
    "<p>Remove downloaded video and images. <b>NOTE: </b>Run this before attempting to split a new video.</p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import shutil\n",
    "shutil.rmtree(\"images\")\n",
    "os.remove(VIDEO_NAME)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<br/>\n",
    "<h1>Install Dependencies</h1>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p>Install the required dependecies required to start using retinanet.</p>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<br/>\n",
    "<h5>Cloning And Install retina-net</h5>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#install retina-net\n",
    "!git clone https://github.com/fizyr/keras-retinanet.git"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<br/>\n",
    "<p>Installing Retina-net</p>\n",
    "<p><b>NOTE:</b> you have to run this command everytime you restart the server.</p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!cd keras-retinanet && pip install . --user"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<br/>\n",
    "<h5>Install other dependecies</h5>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install numpy scipy h5py\n",
    "!pip install scikit-learn Pillow imutils\n",
    "!pip install beautifulsoup4\n",
    "!pip install tensorflow-gpu==1.14\n",
    "# !pip install tensorflow==1.14\n",
    "!pip install keras==2.3.1\n",
    "!pip install opencv-contrib-python\n",
    "!pip install tensorboard\n",
    "!pip install wget\n",
    "!pip install --user git+https://github.com/cocodataset/cocoapi.git#subdirectory=PythonAPI"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<br/>\n",
    "<h1>Building The Dataset</h1>\n",
    "<p>When building a new dataset, you need to run the \"Clear Images And Annotations\" step first, then \"Setting Up Images And Annotations\" and finally \"Creating The train.csv And test.csv\".</p>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "You need to provide the following:\n",
    "<ol>\n",
    "    <li>BUCKET_NAME</li>\n",
    "    <li>S3_IMAGESPATH</li>\n",
    "    <li>S3_ANNOTATIONSPATH</li>\n",
    "</ol>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "BUCKET_NAME = \"csm.calpolydxhub\" #INPUT S3 BUCKET NAME\n",
    "S3_IMAGESPATH = \"images3/\" #INPUT S3 PATH TO IMAGES USED FOR THIS TRAINING JOB\n",
    "S3_ANNOTATIONSPATH = \"output3/\" #INPUT S3 PATH TO ANNOTATIONS USED FOR THIS TRAINING JOB"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<br/>\n",
    "<h5>Create the directory structure for this project and helper files</h5>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p>Build the directories for the project</p>\n",
    "<p><b>NOTE: </b>You don't need to run this step again when building a new dataset.</p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.mkdir(\"config\")\n",
    "os.mkdir(\"dataset\")\n",
    "os.mkdir(\"dataset/annotations\")\n",
    "os.mkdir(\"dataset/images\")\n",
    "os.mkdir(\"dataset/predictions\")\n",
    "os.mkdir(\"models\")\n",
    "os.mkdir(\"snapshots\")\n",
    "os.mkdir(\"tensorboard\")\n",
    "os.mkdir(\"video\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p>Create the python file responsible for providing the paths to all the files</p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "file = open(\"config/esri_retinanet_config.py\",\"w\") \n",
    "\n",
    "file.write(\"# import the necessary packages\\n\") \n",
    "file.write(\"import os\\n\") \n",
    "file.write(\"# initialize the base path for the logos dataset\\n\")\n",
    "file.write(\"BASE_PATH = 'dataset'\\n\")\n",
    "file.write(\"# build the path to the annotations and input images\\n\")\n",
    "file.write(\"ANNOT_PATH = os.path.sep.join([BASE_PATH, 'annotations'])\\n\")\n",
    "file.write(\"IMAGES_PATH = os.path.sep.join([BASE_PATH, 'images'])\\n\")\n",
    "file.write(\"# degine the training/testing split\\n\")\n",
    "file.write(\"TRAIN_TEST_SPLIT = 0.75\\n\")\n",
    "file.write(\"#  build the path to the output training and test .csv files\\n\")\n",
    "file.write(\"TRAIN_CSV = os.path.sep.join([BASE_PATH, 'train.csv'])\\n\")\n",
    "file.write(\"TEST_CSV = os.path.sep.join([BASE_PATH, 'test.csv'])\\n\")\n",
    "file.write(\"# build the path to the output classes CSV files\\n\")\n",
    "file.write(\"CLASSES_CSV = os.path.sep.join([BASE_PATH, 'classes.csv'])\\n\")\n",
    "file.write(\"# build the path to the output predictions dir\\n\")\n",
    "file.write(\"OUTPUT_DIR = os.path.sep.join([BASE_PATH, 'predictions'])\\n\")\n",
    "\n",
    "file.close() \n",
    "\n",
    "file = open(\"config/__init__.py\",\"w\")\n",
    "file.close() "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<br/>\n",
    "<h5>Setting Up Images And Annotations</h5>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p>Set up the function used to download contents of a directory from S3.</p>\n",
    "<p>This function will be used to download the images used for training from S3.</p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import boto3, os\n",
    "import errno\n",
    "s3 = boto3.client('s3')\n",
    "\n",
    "def download_dir(path, target, bucket):\n",
    "    \"\"\"\n",
    "    Downloads recursively the given S3 path to the target directory.\n",
    "    :param path: The S3 directory to download.\n",
    "    :param target: the local directory to download the files to.\n",
    "    :param bucket: the name of the bucket to download from\n",
    "    \"\"\"\n",
    "\n",
    "    # Handle missing / at end of prefix\n",
    "    if not path.endswith('/'):\n",
    "        path += '/'\n",
    "\n",
    "    paginator = s3.get_paginator('list_objects_v2')\n",
    "    for result in paginator.paginate(Bucket=bucket, Prefix=path):\n",
    "        # Download each file individually\n",
    "        for key in result['Contents']:\n",
    "            # Calculate relative path\n",
    "            rel_path = key['Key'][len(path):]\n",
    "            # Skip paths ending in /\n",
    "            if not key['Key'].endswith('/'):\n",
    "                local_file_path = os.path.join(target, rel_path)\n",
    "                # Make sure directories exist\n",
    "                local_file_dir = os.path.dirname(local_file_path)\n",
    "                s3.download_file(bucket, key['Key'], local_file_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Download the images from S3\n",
    "download_dir(S3_IMAGESPATH,\"dataset/images/\",BUCKET_NAME)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p><b>NOTE: </b>After downloading the images, go into the folder containing those images (dataset/images/) and manually delete the .manifest file.</p>\n",
    "<br/>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p>From S3, the following step will get the manifest file containing the data from the labeling job that corresponds with the images downloaded in the previous step</p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "s3_resource = boto3.resource('s3')\n",
    "s3_outputman= s3_resource.Object(BUCKET_NAME, S3_ANNOTATIONSPATH+\"output.manifest\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<br/>\n",
    "<p>We will need to convert the manifest file into one xml file for each image. This is required so that each image is paired with a file that contains its annotation info.</p>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p>For the creation of the xml files to be accomplished, you will need to provide the name of the labeling job from ground truth.</p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "labeling_job_name = \"SMC-labeling-intersection1-2\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Convert the manifest file into a list of json objects\n",
    "import json\n",
    "from io import StringIO\n",
    "\n",
    "manifest_raw= s3_outputman.get()['Body'].read().decode('utf-8')\n",
    "manifest_list = manifest_raw.split(\"\\n\")\n",
    "\n",
    "for i in range(len(manifest_list)-1):\n",
    "    manifest_list[i] = json.load(StringIO(manifest_list[i])) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Make the XML files in the following formate\n",
    "\n",
    "# <?xml version=\"1.0\"?>\n",
    "# <annotation>\n",
    "#     <filename>00001.jpg</filename>\n",
    "#     <source>\n",
    "#         <annotation>ArcGIS Pro 2.1</annotation>\n",
    "#     </source>\n",
    "#     <size>\n",
    "#         <width>1280</width>\n",
    "#         <height>720</height>\n",
    "#         <depth>3</depth>\n",
    "#     </size>\n",
    "#     <object>\n",
    "#         <name>1</name>\n",
    "#         <bndbox>\n",
    "#             <xmin>46.67</xmin>\n",
    "#             <ymin>41.29</ymin>\n",
    "#             <xmax>57.79</xmax>\n",
    "#             <ymax>52.40</ymax>\n",
    "#         </bndbox>\n",
    "#     </object>\n",
    "# </annotation>\n",
    "\n",
    "for i in range(len(manifest_list)-1):\n",
    "    filename = manifest_list[i]['source-ref'].split('/')[-1]\n",
    "    width = manifest_list[i][labeling_job_name]['image_size'][0]['width']\n",
    "    height = manifest_list[i][labeling_job_name]['image_size'][0]['height']\n",
    "    depth = manifest_list[i][labeling_job_name]['image_size'][0]['depth']\n",
    "    labels = manifest_list[i][labeling_job_name]['annotations']\n",
    "    labels_num = len(labels)\n",
    "\n",
    "    xml_file = open(\"dataset/annotations/\"+filename[0:-3]+\"xml\",\"w\")\n",
    "    xml_file.write('<?xml version=\"1.0\"?>\\n')\n",
    "    xml_file.write('<annotation>\\n')\n",
    "    xml_file.write('<filename>'+filename+'</filename>\\n')\n",
    "    xml_file.write('<source>\\n')\n",
    "    xml_file.write('<annotation>SMC-Scooter-Vision</annotation>\\n')\n",
    "    xml_file.write('</source>\\n')\n",
    "    xml_file.write('<size>\\n')\n",
    "    xml_file.write('<width>'+str(width)+'</width>\\n')\n",
    "    xml_file.write('<height>'+str(height)+'</height>\\n')\n",
    "    xml_file.write('<depth>'+str(depth)+'</depth>\\n')\n",
    "    xml_file.write('</size>\\n')\n",
    "\n",
    "    for i in range(labels_num):\n",
    "        label_type = labels[i]['class_id']\n",
    "        label_width = labels[i]['width']\n",
    "        label_top = labels[i]['top']\n",
    "        label_height = labels[i]['height']\n",
    "        label_left = labels[i]['left']\n",
    "        xml_file.write('<object>\\n')\n",
    "        xml_file.write('<name>'+str(label_type)+'</name>\\n')\n",
    "        xml_file.write('<bndbox>\\n')\n",
    "        xml_file.write('<xmin>'+str(label_left)+'</xmin>\\n')\n",
    "        xml_file.write('<ymin>'+str(label_top)+'</ymin>\\n')\n",
    "        xml_file.write('<xmax>'+str(label_left+label_width)+'</xmax>\\n')\n",
    "        xml_file.write('<ymax>'+str(label_top+label_height)+'</ymax>\\n')\n",
    "        xml_file.write('</bndbox>\\n')\n",
    "        xml_file.write('</object>\\n')\n",
    "    xml_file.write('</annotation>\\n')\n",
    "\n",
    "    xml_file.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<br/>\n",
    "<h5>Creating The train.csv And test.csv</h5>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import the necessary packages\n",
    "!pip install imutils\n",
    "from config import config\n",
    "from bs4 import BeautifulSoup\n",
    "from imutils import paths\n",
    "import argparse\n",
    "import random\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "annot_path = config.ANNOT_PATH\n",
    "images_path = config.IMAGES_PATH\n",
    "train_csv = config.TRAIN_CSV\n",
    "test_csv = config.TEST_CSV\n",
    "classes_csv = config.CLASSES_CSV\n",
    "train_test_split = config.TRAIN_TEST_SPLIT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# grab all image paths then construct the training and testing split\n",
    "from os import listdir\n",
    "from os.path import isfile, join\n",
    "imagePaths = [f for f in listdir(images_path) if isfile(join(images_path, f))]\n",
    "# random.shuffle(imagePaths)\n",
    "i = int(len(imagePaths) * train_test_split)\n",
    "trainImagePaths = imagePaths[:i]\n",
    "testImagePaths = imagePaths[i:]\n",
    "\n",
    "# create the list of datasets to build\n",
    "dataset = [ (\"train\", trainImagePaths, train_csv),\n",
    "            (\"test\", testImagePaths, test_csv)]\n",
    "\n",
    "# initialize the set of classes we have\n",
    "CLASSES = set()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# loop over the datasets\n",
    "for (dType, imagePaths, outputCSV) in dataset:\n",
    "    # load the contents\n",
    "    print (\"[INFO] creating '{}' set...\".format(dType))\n",
    "    print (\"[INFO] {} total images in '{}' set\".format(len(imagePaths), dType))\n",
    "\n",
    "    # open the output CSV file\n",
    "    csv = open(outputCSV, \"w\")\n",
    "\n",
    "    # loop over the image paths\n",
    "    for imagePath in imagePaths:\n",
    "        # build the corresponding annotation path\n",
    "        fname = imagePath.split(os.path.sep)[0]\n",
    "        fname = \"{}.xml\".format(fname[:fname.rfind(\".\")])\n",
    "        annotPath = os.path.sep.join([annot_path, fname])\n",
    "\n",
    "        # load the contents of the annotation file and buid the soup\n",
    "        contents = open(annotPath).read()\n",
    "        soup = BeautifulSoup(contents, \"html.parser\")\n",
    "\n",
    "        # extract the image dimensions\n",
    "        w = int(soup.find(\"width\").string)\n",
    "        h = int(soup.find(\"height\").string)\n",
    "        \n",
    "        # loop over all object elements\n",
    "        for o in soup.find_all(\"object\"):\n",
    "            #extract the label and bounding box coordinates\n",
    "            label = o.find(\"name\").string\n",
    "            xMin = int(float(o.find(\"xmin\").string))\n",
    "            yMin = int(float(o.find(\"ymin\").string))\n",
    "            xMax = int(float(o.find(\"xmax\").string))\n",
    "            yMax = int(float(o.find(\"ymax\").string))\n",
    "\n",
    "            # truncate any bounding box coordinates that fall outside\n",
    "            # the boundaries of the image\n",
    "            xMin = max(0, xMin)\n",
    "            yMin = max(0, yMin)\n",
    "            xMax = min(w, xMax)\n",
    "            yMax = min(h, yMax)\n",
    "\n",
    "            # ignore the bounding boxes where the minimum values are larger\n",
    "            # than the maximum values and vice-versa due to annotation errors\n",
    "            if xMin >= xMax or yMin >= yMax:\n",
    "                continue\n",
    "            elif xMax <= xMin or yMax <= yMin:\n",
    "                continue\n",
    "\n",
    "            # write the image path, bb coordinates, label to the output CSV\n",
    "            row = [os.path.abspath('dataset/images'+'/'+imagePath),str(xMin), str(yMin), str(xMax),str(yMax), str(label)]\n",
    "            csv.write(\"{}\\n\".format(\",\".join(row)))\n",
    "\n",
    "            # update the set of unique class labels\n",
    "            CLASSES.add(label)\n",
    "\n",
    "    # close the CSV file\n",
    "    csv.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# write the classes to file\n",
    "print(\"[INFO] writing classes...\")\n",
    "csv = open(classes_csv, \"w\")\n",
    "rows = [\",\".join([c, str(i)]) for (i,c) in enumerate(CLASSES)]\n",
    "csv.write(\"\\n\".join(rows))\n",
    "csv.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<br/>\n",
    "<h5>Clear Images And Annotations</h5>\n",
    "<p><b>NOTE: </b>Run this before attempting to build a new dataset</p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import shutil\n",
    "shutil.rmtree(\"dataset/images\")\n",
    "shutil.rmtree(\"dataset/annotations\")\n",
    "os.mkdir(\"dataset/images\")\n",
    "os.mkdir(\"dataset/annotations\")\n",
    "os.remove(\"dataset/train.csv\")\n",
    "os.remove(\"dataset/test.csv\")\n",
    "os.remove(\"dataset/classes.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<br/>\n",
    "<h1>Training</h1>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h5>Setting up for training</h5>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p>Download a pre-trained model to use as backbone</p>\n",
    "<p><b>NOTE:</b> Don't run this if you already have a base model. A base model exists in the github repository under models, download it then upload it to the model folder created in the previous steps</p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import wget\n",
    "url = 'https://github.com/fizyr/keras-retinanet/releases/download/0.5.1/resnet50_coco_best_v2.1.0.h5'\n",
    "wget.download(url, 'models/resnet50_coco_best_v2.1.0.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p>Set number of steps and epochs</p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rows = !wc -l dataset/train.csv\n",
    "rows = int(rows[0].split()[0])\n",
    "batch_size = 4;\n",
    "steps = rows//batch_size\n",
    "epochs = 40"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p>Begin training</p>\n",
    "<p><b>NOTE: </b>Don't forget to use the previous model if doing more training.</p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "# previousModel = \"resnet50_coco_best_v2.1.0.h5\"\n",
    "previousModel = \"model_12_20_2019.h5\"\n",
    "trainingNumber = len(next(os.walk('snapshots'))[1])-1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!retinanet-train --weights models/$previousModel \\\n",
    "--batch-size $batch_size --steps $steps --epochs $epochs \\\n",
    "--snapshot-path snapshots/$trainingNumber --tensorboard-dir tensorboard/$trainingNumber \\\n",
    "csv dataset/train.csv dataset/classes.csv"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h5>Analyze the trained model with tensorboard</h5>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "latest_tensorboard = trainingNumber - 1;\n",
    "!tensorboard --logdir=tensorboard/$latest_tensorboard"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<br/>\n",
    "<h1>Convert Model To An Infrence Model</h1>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h5>Convert the model</h5>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import datetime\n",
    "\n",
    "# getting current date and time\n",
    "today = datetime.datetime.today()\n",
    "new_modelName = 'model_'+str(today.month)+'_'+str(today.day)+'_'+str(today.year)+'.h5'\n",
    "print('model name: ',new_modelName)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Get the latest snapshot\n",
    "import glob\n",
    "latest_snapshotFolder = trainingNumber - 1;\n",
    "list_of_snapshot = glob.glob('snapshots/'+str(latest_snapshotFolder)+'/*.h5') # * means all if need specific format then *.csv\n",
    "latest_snapshot = max(list_of_snapshot, key=os.path.getctime)\n",
    "\n",
    "#Start training\n",
    "!retinanet-convert-model $latest_snapshot models/$new_modelName"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h5>Evaluate the model</h5>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "modelName = 'model_2_17_2020.h5' #Name of the model to evaluate\n",
    "!retinanet-evaluate csv dataset/train.csv dataset/classes.csv models/$modelName"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<br/>\n",
    "<h1>Infrence</h1>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p><b>NOTE: </b>If you have a problem with dependencies here, go back and run the \"Install Retina-net\" step again. Make sure to only install and not clone again.</p>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h5>Load necessary modules</h5>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# show images inline\n",
    "%matplotlib inline\n",
    "\n",
    "# automatically reload modules when they have changed\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "# import keras\n",
    "import keras\n",
    "\n",
    "# import keras_retinanet\n",
    "from keras_retinanet import models\n",
    "from keras_retinanet.utils.image import read_image_bgr, preprocess_image, resize_image\n",
    "from keras_retinanet.utils.visualization import draw_box, draw_caption\n",
    "from keras_retinanet.utils.colors import label_color\n",
    "from keras_retinanet.utils.gpu import setup_gpu\n",
    "\n",
    "# import miscellaneous modules\n",
    "import matplotlib.pyplot as plt\n",
    "import cv2\n",
    "import os\n",
    "import numpy as np\n",
    "import time\n",
    "\n",
    "# use this to change which GPU to use\n",
    "gpu = 0\n",
    "\n",
    "# set the modified tf session as backend in keras\n",
    "setup_gpu(gpu)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_path = os.path.join('models', 'model_2_17_2020.h5')\n",
    "\n",
    "# load retinanet model\n",
    "model = models.load_model(model_path, backbone_name='resnet50')\n",
    "\n",
    "labels_to_names = {0: 'sidewalk', 1: 'street'}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h5>Run predicition on an image</h5>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p><b>Green</b> labels represnets scooters on the street and <b>Red</b> labels are for scooters on the sidewalk</p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict(imagePath, saveDest, displayImage):\n",
    "    image = read_image_bgr(imagePath)\n",
    "    imageName = imagePath.split('/')[-1]\n",
    "    boxLabeles = []\n",
    "\n",
    "    # copy to draw on\n",
    "    draw = image.copy()\n",
    "    draw = cv2.cvtColor(draw, cv2.COLOR_BGR2RGB)\n",
    "\n",
    "    # preprocess image for network\n",
    "    image = preprocess_image(image)\n",
    "    image, scale = resize_image(image)\n",
    "\n",
    "    # process image\n",
    "    start = time.time()\n",
    "    boxes, scores, labels = model.predict_on_batch(np.expand_dims(image, axis=0))\n",
    "    if(displayImage):\n",
    "        print(\"processing time: \", time.time() - start)\n",
    "\n",
    "    # correct for image scale\n",
    "    boxes /= scale\n",
    "\n",
    "    scooters_num = 0\n",
    "    # visualize detections\n",
    "    for box, score, label in zip(boxes[0], scores[0], labels[0]):\n",
    "        # scores are sorted so we can break\n",
    "        if score < 0.20:\n",
    "            break\n",
    "            \n",
    "        scooters_num +=1\n",
    "        if(displayImage):\n",
    "            print(labels_to_names[label],\": \",score)        \n",
    "        \n",
    "        #+5 changes the color\n",
    "        #RED: SIDEWALK\n",
    "        #GREEN: Street\n",
    "        color = label_color(label+5)\n",
    "\n",
    "        b = box.astype(int)\n",
    "        draw_box(draw, b, color=color)  \n",
    "        draw_box(draw, (1050,250,1120,400), color=color)  \n",
    "        boxLabeles.append((b[0], b[1], b[2]-b[0], b[3]-b[1]))\n",
    "    \n",
    "        caption = \"{} {:.3f}\".format(labels_to_names[label], score)\n",
    "        draw_caption(draw, b, caption)\n",
    "\n",
    "    if(displayImage):\n",
    "        plt.figure(figsize=(20, 20))\n",
    "        plt.axis('off')\n",
    "        plt.imshow(draw)\n",
    "        plt.show()\n",
    "    cv2.imwrite(saveDest+'/'+imageName, cv2.cvtColor(draw, cv2.COLOR_BGR2RGB))\n",
    "    print(scooters_num)\n",
    " \n",
    "    return boxLabeles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load image\n",
    "# imageName = 'dataset/images/00132.jpg'\n",
    "# imageName = 'dataset/images/00715'\n",
    "imageName = '2.jpg'\n",
    "predict(imageName, 'dataset/predictions', True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<br/>\n",
    "<h1>Sidewalk Detection</h1>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p>Import dependencies</p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install fastai"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai.vision.models import *\n",
    "from fastai.vision.learner import *\n",
    "from fastai.vision import *\n",
    "from fastai.vision import Image"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p>Test Dataset</p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "path=Path('camvid_tiny') #setup path\n",
    "path_lbl = path/'labels'\n",
    "path_img = path/'images'\n",
    "fnames = get_image_files(path_img)\n",
    "fnames[:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "img_f = fnames[0]\n",
    "img = open_image(img_f)\n",
    "img.show(figsize=(5,5))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p>Test Labels</p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lbl_names = get_image_files(path_lbl)\n",
    "lbl_names[:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "get_y_fn = lambda x: path_lbl/f'{x.stem}_P{x.suffix}'\n",
    "mask = open_mask(get_y_fn(img_f))\n",
    "mask.show(figsize=(5,5), alpha=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "src_size = np.array(mask.shape[1:])   #mask data\n",
    "src_size,mask.data\n",
    "codes = np.loadtxt(path/'codes.txt', dtype=str)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p>Create Dataloader</p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = (SegmentationItemList.from_folder(path_img)\n",
    "        .split_by_rand_pct()\n",
    "        .label_from_func(get_y_fn, classes=codes)\n",
    "        .transform(get_transforms(), tfm_y=True, size=256)\n",
    "        .databunch(bs=2, path=path)\n",
    "        .normalize(imagenet_stats))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "name2id = {v:k for k,v in enumerate(codes)}\n",
    "void_code = name2id['Void']\n",
    "\n",
    "def acc_camvid(input, target):     #defining accuracy\n",
    "    target = target.squeeze(1)\n",
    "    mask = target != void_code\n",
    "    return (input.argmax(dim=1)[mask]==target[mask]).float().mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn=unet_learner(data,models.resnet34, metrics=acc_camvid) #creating the architecture with imagenet weights\n",
    "learn.data.single_ds.tfmargs['size'] = None"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h5>Inference</h5>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.load('stage-2')   #Loading the trained Model in this case it is stage-2 modek which is stored under camvid-tiny/models\n",
    "def infer(x):\n",
    "    #returrns [ROW][COL]\n",
    "    start = time.time()\n",
    "    img = open_image(x)\n",
    "    c=learn.predict(img)\n",
    "    print(c[0])\n",
    "    print(c[1][0][0][0])\n",
    "    c[0].show()\n",
    "    print(\"processing time: \", time.time() - start)\n",
    "    return c[1][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "table = infer('2.jpg')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h5>Functions</h5>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sidewalkOverlap(sidewalkTable, bbox):\n",
    "    #sidewalkTable e.g. [ROW][COL]\n",
    "    # bbox. e.g. (730, 170, 20, 40)\n",
    "    #returns \"street\" or \"sidewalk\"\n",
    "    \n",
    "    #Percent of area to check for overlap with sidewalk starting from the bottom of the bounding box\n",
    "    percentArea = .25\n",
    "    \n",
    "    #What percentage of scooter pixels in the checked area should determine a sidewalk overlap\n",
    "    overlapThreshold = .20\n",
    "    \n",
    "    startX = bbox[0] - 1\n",
    "    startY = bbox[1] - 1\n",
    "    endX = bbox[0]+bbox[2] -1\n",
    "    endY = bbox[1]+bbox[3] -1\n",
    "    \n",
    "    #How many columns are being checked\n",
    "    colQty = endX-startX\n",
    "    \n",
    "    #Shortened sidewalk table\n",
    "    table = sidewalkTable[startY:endY]\n",
    "    \n",
    "    #Area of the portion used to calculate overlap\n",
    "    area = colQty*(endY-startY)*percentArea\n",
    "    \n",
    "    rowCounter = 0\n",
    "    sidewalkCounter = 0\n",
    "    for index in range(len(table)):\n",
    "        rowCounter+=1\n",
    "        row = table[-(index+1)]\n",
    "        newRow = row[startX:endX]\n",
    "    \n",
    "        for col in newRow:\n",
    "            if col == 19:\n",
    "                sidewalkCounter+=1\n",
    "            \n",
    "        if(rowCounter >= colQty*percentArea):\n",
    "            break\n",
    "    \n",
    "    percentOverlap = sidewalkCounter/area\n",
    "#     print(area)\n",
    "#     print(sidewalkCounter)\n",
    "#     print(percentOverlap)\n",
    "\n",
    "    if(percentOverlap>=overlapThreshold):\n",
    "        return \"sidewalk\"\n",
    "    return \"street\"\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sidewalkOverlap(table,(1050,250,1120,400))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<br/>\n",
    "<h1>Label A Video</h1>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b>NOTE:</b> This section does not involve any tracking and counting of scooters. It's only here to test the detection model."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p>Download the video you want labeled from s3.</p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import boto3, os\n",
    "s3 = boto3.client('s3')\n",
    "\n",
    "BUCKET_NAME = \"csm.calpolydxhub\" #INPUT S3 BUCKET NAME\n",
    "VIDEO_NAME = \"scooter_Footage_pico.mp4\"#INPUT VIDEO NAME\n",
    "VIDEO_DIR = \"video/filtered videos/\"+VIDEO_NAME\n",
    "\n",
    "video = s3.download_file(BUCKET_NAME, VIDEO_DIR, VIDEO_NAME)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<br/>\n",
    "<p>Specify the name of the video to be labeled and what it should be named.</p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "video_path = 'video/scooter_Footage_pico2.mp4'\n",
    "labeledVideo_path = 'video/labeledVideo2.mp4'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cap = cv2.VideoCapture(video_path)\n",
    "fourcc = cv2.VideoWriter_fourcc(*'XVID')\n",
    "\n",
    "frameRate = cap.get(5) #get frame rate of video\n",
    "totalFrames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))\n",
    "ret, frame = cap.read()\n",
    "height, width, layers = frame.shape\n",
    "size = (width,height)\n",
    "out = cv2.VideoWriter(labeledVideo_path,fourcc, frameRate, size)\n",
    "\n",
    "frame_number = 0\n",
    "\n",
    "while (cap.isOpened()) :\n",
    "    ret,frame = cap.read()\n",
    "    if ret != True:\n",
    "        break;\n",
    "        \n",
    "    frame_number += 1\n",
    "    \n",
    "    if frame_number % 2 == 1:\n",
    "        new_frame = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR)\n",
    "        # preprocess image for network\n",
    "        network_frame = preprocess_image(new_frame)\n",
    "        network_frame, scale = resize_image(new_frame)\n",
    "\n",
    "        boxes, scores, labels = model.predict_on_batch(np.expand_dims(network_frame, axis=0))\n",
    "\n",
    "        # correct for image scale\n",
    "        boxes /= scale\n",
    "\n",
    "        # visualize detections\n",
    "        for box, score, label in zip(boxes[0], scores[0], labels[0]):\n",
    "            # scores are sorted so we can break\n",
    "            if score < 0.70:\n",
    "                break\n",
    "            #+5 changes the color\n",
    "            #RED: SIDEWALK\n",
    "            #GREEN: Street\n",
    "            color = label_color(label+5)\n",
    "\n",
    "            b = box.astype(int)\n",
    "            draw_box(new_frame, b, color=color)\n",
    "\n",
    "            caption = \"{} {:.3f}\".format(labels_to_names[label], score)\n",
    "            draw_caption(new_frame, b, caption)\n",
    "\n",
    "        out.write(cv2.cvtColor(new_frame, cv2.COLOR_BGR2RGB))\n",
    "        print(\"\\r\", \"{}/{} frames completed\".format(frame_number, totalFrames), end=\"\")\n",
    "        \n",
    "cap.release()\n",
    "out.release()\n",
    "cv2.destroyAllWindows()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<br/>\n",
    "<h1>Count Scooters</h1>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p>This section will utilize the ML model to count the number of scooters on/off sidewalk from a submitted video</p>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<br/>\n",
    "<h1>Reduce Video Length</h1>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import boto3, os\n",
    "s3 = boto3.client('s3')\n",
    "\n",
    "#S3 Bucket directory and video name\n",
    "BUCKET_NAME = \"csm.calpolydxhub\" #INPUT S3 BUCKET NAME\n",
    "VIDEO_NAME = \"broadway-3rd.mpg\"#INPUT VIDEO NAME\n",
    "VIDEO_DIR = \"video/20190919/\"+VIDEO_NAME"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Download the video to this directory\n",
    "video = s3.download_file(BUCKET_NAME, VIDEO_DIR, VIDEO_NAME)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "cap = cv2.VideoCapture(VIDEO_NAME)\n",
    "\n",
    "frameRate = cap.get(5) #get frame rate of video\n",
    "Frames_Per_Second = .5\n",
    "totalFrames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))\n",
    "\n",
    "PADDING = round(frameRate)* 2 #two seconds before and after the detection\n",
    "\n",
    "frame_number = 0\n",
    "write_range = 0\n",
    "write_threshold = PADDING\n",
    "newFrames_list = [1]\n",
    "while (cap.isOpened()):\n",
    "    ret,frame = cap.read()\n",
    "    if ret != True:\n",
    "        break;\n",
    "        \n",
    "    frame_number += 1\n",
    "    scooter_num = 0\n",
    "    write_range -= 1\n",
    "    if ( (frame_number % math.floor(frameRate//Frames_Per_Second) == 0) and (write_range <= 0) ):\n",
    "        new_frame = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR)\n",
    "        # preprocess image for network\n",
    "        network_frame = preprocess_image(new_frame)\n",
    "        network_frame, scale = resize_image(new_frame)\n",
    "\n",
    "        boxes, scores, labels = model.predict_on_batch(np.expand_dims(network_frame, axis=0))\n",
    "\n",
    "        # correct for image scale\n",
    "        boxes /= scale\n",
    "\n",
    "        # visualize detections\n",
    "        for box, score, label in zip(boxes[0], scores[0], labels[0]):\n",
    "            # scores are sorted so we can break\n",
    "            if score < 0.50:\n",
    "                break\n",
    "            scooter_num += 1;\n",
    "        if(scooter_num != 0):\n",
    "            write_range = PADDING\n",
    "            for i in range(-1*write_threshold, write_threshold+1):\n",
    "                frame_num = frame_number + i\n",
    "                if(frame_num <= 0):\n",
    "                    continue\n",
    "                if(not newFrames_list[-1] >= frame_num):\n",
    "                    newFrames_list.append(frame_num)\n",
    "    print(\"\\r\", \"{}/{} frames completed\".format(frame_number, totalFrames), end=\"\")    \n",
    "\n",
    "cap.release()\n",
    "cv2.destroyAllWindows()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print( \"Video will be reduced to \",(len(newFrames_list)/totalFrames)*100, \" of the original length\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cap = cv2.VideoCapture(VIDEO_NAME)\n",
    "fourcc = cv2.VideoWriter_fourcc(*'XMPG') #refrence this doc: http://www.fourcc.org/codecs.php if using a different video format\n",
    "\n",
    "frameRate = cap.get(5) #get frame rate of video\n",
    "frameCount = len(newFrames_list)\n",
    "ret, frame = cap.read()\n",
    "height, width, layers = frame.shape\n",
    "size = (width,height)\n",
    "out = cv2.VideoWriter(\"MLREDUCED\"+VIDEO_NAME,fourcc, frameRate, size)\n",
    "\n",
    "frame_number = 0\n",
    "current_frame = 0\n",
    "while (cap.isOpened()):\n",
    "    ret,frame = cap.read()\n",
    "    if ret != True:\n",
    "        break;\n",
    "    frame_number += 1\n",
    "    \n",
    "    if frame_number == newFrames_list[current_frame]:\n",
    "        current_frame +=1\n",
    "        out.write(frame)\n",
    "    print(\"\\r\", \"{}/{} frames completed\".format(current_frame, frameCount), end=\"\") \n",
    "        \n",
    "cap.release()\n",
    "out.release()\n",
    "cv2.destroyAllWindows()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<br/>\n",
    "<h1>TRACKING</h1>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h5>Initialize helper functions and classes</h5>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import cv2\n",
    "import sys\n",
    "import imutils\n",
    "from PIL import Image as PImage\n",
    "\n",
    "scooters = []\n",
    "sidewalkTable = None\n",
    "counter_street = 0\n",
    "counter_sidewalk = 0\n",
    "class Tracker:\n",
    "    Id = None\n",
    "    tracker = None\n",
    "    box=[]\n",
    "    status=False\n",
    "    labelType = None\n",
    "    correctionByDetector = False\n",
    "    attemptedToRemove = False\n",
    "    \n",
    "    def __init__(self, frame, bbox):\n",
    "        global counter_street\n",
    "        global counter_sidewalk \n",
    "        self.Id = counter_street\n",
    "        self.tracker = cv2.TrackerCSRT_create()\n",
    "        self.box = bbox;\n",
    "        self.status = self.tracker.init(frame, bbox)\n",
    "        self.labelType = sidewalkOverlap(sidewalkTable, bbox)\n",
    "        if(self.labelType == \"street\"):\n",
    "            counter_street +=1\n",
    "        else:\n",
    "            counter_sidewalk +=1\n",
    "        self.correctionByDetector = True\n",
    "        \n",
    "    def updateTracking(self, frame):\n",
    "        self.status, self.box = self.tracker.update(frame)\n",
    "        self.correctionByDetector = False\n",
    "        if not self.status:\n",
    "            self.attemptedToRemove = True\n",
    "            self.removeSelf() \n",
    "            \n",
    "    def reinitialze(self, frame, bbox):\n",
    "        global counter_street\n",
    "        global counter_sidewalk\n",
    "        self.correctionByDetector = True\n",
    "        self.tracker = cv2.TrackerCSRT_create()\n",
    "        self.box = bbox;\n",
    "        self.status = self.tracker.init(frame, bbox)\n",
    "        newLabel = sidewalkOverlap(sidewalkTable, bbox)   \n",
    "        #If scooter switches from street to sidewalk, change its count to sidewalk\n",
    "        if(self.labelType != newLabel):\n",
    "            if(self.labelType == \"street\"):\n",
    "                counter_street -=1\n",
    "                counter_sidewalk +=1\n",
    "        self.labelType = newLabel\n",
    "            \n",
    "    def removeSelf(self):\n",
    "        if (self.attemptedToRemove):\n",
    "            scooters.remove(self)#remove current instance from objects list\n",
    "        else:\n",
    "            self.attemptedToRemove = True\n",
    "            \n",
    "    def writeFrame(self, frame):\n",
    "        if self.status:\n",
    "            p1 = (int(self.box[0]), int(self.box[1]))\n",
    "            p2 = (int(self.box[0] + self.box[2]), int(self.box[1] + self.box[3]))\n",
    "            if(self.labelType == 'street'):\n",
    "                cv2.rectangle(frame, p1, p2, (255,0,0), 2, 1)\n",
    "            else:\n",
    "                cv2.rectangle(frame, p1, p2, (255,255,0), 2, 1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "HIGH_CONFIDENCE_THRESHOLD=.95\n",
    "LOW_CONFIDENCE_THRESHOLD=.2\n",
    "\n",
    "def inferSidewalk(frame):\n",
    "    #returrns [ROW][COL]\n",
    "    new_frame = cv2.cvtColor(frame,cv2.COLOR_BGR2RGB)\n",
    "    pil_img = PImage.fromarray(new_frame.astype('uint8'), 'RGB')\n",
    "    pil_img = pil2tensor(pil_img,np.float32)\n",
    "    inf_img = Image(pil_img.div_(255))\n",
    "    c=learn.predict(inf_img)\n",
    "    return c[1][0]\n",
    "    \n",
    "def detector(frame):\n",
    "    highConfidenceBoxes=[]\n",
    "    lowConfidenceBoxes=[]\n",
    "    new_frame = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR)\n",
    "    # preprocess image for network\n",
    "    network_frame = preprocess_image(new_frame)\n",
    "    network_frame, scale = resize_image(new_frame)\n",
    "    boxes, scores, labels = model.predict_on_batch(np.expand_dims(network_frame, axis=0))\n",
    "\n",
    "    boxes /= scale\n",
    "    detectorHash={}\n",
    "    # visualize detections\n",
    "    for box, score, label in zip(boxes[0], scores[0], labels[0]):\n",
    "        if score<LOW_CONFIDENCE_THRESHOLD:\n",
    "            break\n",
    "        bx = box.astype(int)\n",
    "        bbox = (bx[0], bx[1], bx[2]-bx[0], bx[3]-bx[1])\n",
    "        tup= (bbox,labels_to_names[label],score)\n",
    "        \n",
    "        if score>=HIGH_CONFIDENCE_THRESHOLD:\n",
    "            highConfidenceBoxes.append(tup)\n",
    "            color = label_color(label+5)\n",
    "            draw_box(frame, bx, color=color)\n",
    "        else:\n",
    "            if len(highConfidenceBoxes)==0:\n",
    "                lowConfidenceBoxes.append(tup)   \n",
    "            else:\n",
    "                doesOverlap = False\n",
    "                for HCbox in highConfidenceBoxes:\n",
    "                    if overlap(HCbox[0],tup[0], False) >=.8: #Filter overlaped boxes\n",
    "                        doesOverlap = True\n",
    "                        break\n",
    "                if not doesOverlap:\n",
    "                    color = label_color(label+14)\n",
    "                    draw_box(frame, bx, color=color)\n",
    "                    lowConfidenceBoxes.append(tup)\n",
    "    \n",
    "    detectorHash['high_confidence']=highConfidenceBoxes\n",
    "    detectorHash['low_confidence']=highConfidenceBoxes+lowConfidenceBoxes\n",
    "    return detectorHash\n",
    "\n",
    "def overlap(i, j, isTracker):\n",
    "    #box1\n",
    "    bb1_x1 = i[0]\n",
    "    bb1_x2 = i[0] + i[2]\n",
    "    bb1_y1 = i[1]\n",
    "    bb1_y2 = i[1] + i[3]\n",
    "    \n",
    "    #box2\n",
    "    if isTracker:\n",
    "        bb2_x1 = j.box[0]\n",
    "        bb2_x2 = j.box[0] + j.box[2]\n",
    "        bb2_y1 = j.box[1]\n",
    "        bb2_y2 = j.box[1] + j.box[3]\n",
    "    else:\n",
    "        bb2_x1 = j[0]\n",
    "        bb2_x2 = j[0] + j[2]\n",
    "        bb2_y1 = j[1]\n",
    "        bb2_y2 = j[1] + j[3]\n",
    "        \n",
    "    if bb1_x1 >= bb1_x2 or bb1_y1 >= bb1_y2 or bb2_x1 >= bb2_x2 or bb2_y1 >= bb2_y2:\n",
    "        return -1\n",
    "    \n",
    "  \n",
    "    # determine the coordinates of the intersection rectangle\n",
    "    x_left = max(bb1_x1, bb2_x1)\n",
    "    y_top = max(bb1_y1, bb2_y1)\n",
    "    x_right = min(bb1_x2, bb2_x2)\n",
    "    y_bottom = min(bb1_y2, bb2_y2)\n",
    "\n",
    "    if x_right < x_left or y_bottom < y_top:\n",
    "        return 0.0\n",
    "    \n",
    "    # The intersection of two axis-aligned bounding boxes is always an\n",
    "    # axis-aligned bounding box\n",
    "    intersection_area = (x_right - x_left+1) * (y_bottom - y_top+1)\n",
    "\n",
    "    # compute the area of both AABBs\n",
    "    bb1_area = (bb1_x2 - bb1_x1+1) * (bb1_y2 - bb1_y1+1)\n",
    "    bb2_area = (bb2_x2 - bb2_x1+1) * (bb2_y2 - bb2_y1+1)\n",
    "\n",
    "#     iou = intersection_area / float(bb1_area + bb2_area - intersection_area)\n",
    "    if intersection_area == bb2_area:\n",
    "        return 1\n",
    "    \n",
    "    overlap = intersection_area/bb1_area\n",
    "    return overlap\n",
    "\n",
    "def filterbox(frame,detectorHash):\n",
    "    highConfDetectorBoxes = detectorHash['high_confidence']\n",
    "    lowConfDetectorBoxes = detectorHash['low_confidence']\n",
    "    \n",
    "    #checks to see if the overlap is significant\n",
    "    if len(scooters) == 0:\n",
    "        for i in highConfDetectorBoxes:\n",
    "            scooters.append( Tracker( frame, i[0]) )\n",
    "    else:\n",
    "        for i in lowConfDetectorBoxes:\n",
    "            referenced = False #has this box been referenced to an existing tracker\n",
    "\n",
    "            for j in scooters:\n",
    "                if j.correctionByDetector:\n",
    "                    continue\n",
    "                if overlap(i[0], j, True)>0.5:\n",
    "                    j.reinitialze(frame, i[0])\n",
    "                    referenced = True\n",
    "                    break\n",
    "\n",
    "            if not(referenced) and (i[2]>=HIGH_CONFIDENCE_THRESHOLD):\n",
    "                scooters.append( Tracker( frame, i[0] ) )\n",
    "                \n",
    "                \n",
    "        for j in scooters: #remove scooter trackers that weren't detected by the scooter\n",
    "            if j.correctionByDetector:\n",
    "                continue\n",
    "            j.removeSelf()\n",
    "                \n",
    "                    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h5>Run program</h5>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Insert video name below\n",
    "VIDEO_ADDRESS= 'video/scooter_Footage_pico.mp4'\n",
    "# VIDEO_ADDRESS= 'broadway-3rd.mpg'\n",
    "\n",
    "#Insert output video name below\n",
    "OUTPUTVIDEO_ADDRESS= 'video/intersection3_cut_TRACKED.mp4'\n",
    "\n",
    "video = cv2.VideoCapture(VIDEO_ADDRESS)\n",
    "\n",
    "if not video.isOpened():\n",
    "    print (\"Could not open video\")\n",
    "    sys.exit()\n",
    "\n",
    "frameRate = video.get(5) #get frame rate of video\n",
    "ret, frame = video.read()\n",
    "height, width, layers = frame.shape\n",
    "size = (width,height)\n",
    "totalFramesInVideo = int(video.get(cv2.CAP_PROP_FRAME_COUNT))\n",
    "\n",
    "\n",
    "#Initialize output video\n",
    "fourcc = cv2.VideoWriter_fourcc(*'XVID')\n",
    "output_video = cv2.VideoWriter(OUTPUTVIDEO_ADDRESS,fourcc, frameRate, size)\n",
    "\n",
    "# Exit if video not opened.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#The number of frames to run the detector after\n",
    "detectorRoutine = 30\n",
    "sidewalkDetectionRoutine = 5000\n",
    "\n",
    "while (video.isOpened()):\n",
    "    frameId = video.get(1) #current frame number\n",
    "    print(\"\\r\", \"{}/{} frames completed\".format(frameId, totalFramesInVideo), end=\"\")  \n",
    "    \n",
    "    # Read a new frame\n",
    "    ok, frame = video.read()\n",
    "    \n",
    "    if not ok:\n",
    "        break\n",
    "\n",
    "    timer = cv2.getTickCount()\n",
    "    \n",
    "    # get updated location of objects in subsequent frames\n",
    "    for scooter in scooters:\n",
    "        scooter.updateTracking(frame)\n",
    "\n",
    "    #run sidewalk detector on routine\n",
    "    if(frameId-1) % sidewalkDetectionRoutine == 0:\n",
    "        sidewalkTable = inferSidewalk(frame)\n",
    "        \n",
    "    #run the detector on routine or when an object is lost\n",
    "    if (frameId-1) % detectorRoutine == 0:\n",
    "        bboxes = detector(frame); #get new bounding boxes and label types as tuples from detector model\n",
    "        filterbox(frame, bboxes); #filter any overlaps in the objects on the frame\n",
    "        \n",
    "\n",
    "    fps = cv2.getTickFrequency() / (cv2.getTickCount() - timer)\n",
    "    \n",
    "    for scooter in scooters:\n",
    "        scooter.writeFrame(frame)\n",
    "    \n",
    "    # Display tracker type on frame\n",
    "    cv2.putText(frame,\"Street Count: \" + str(int(counter_street)), (100,20), cv2.FONT_HERSHEY_SIMPLEX, 0.75, (0,0,0),2);\n",
    "    cv2.putText(frame,\"Sidewalk Count: \" + str(int(counter_sidewalk)), (100,50), cv2.FONT_HERSHEY_SIMPLEX, 0.75, (0,0,0),2);\n",
    "    cv2.putText(frame, \"FPS : \" + str(int(fps)), (100,80), cv2.FONT_HERSHEY_SIMPLEX, 0.75, (255,255,255), 2);\n",
    "    \n",
    "    output_video.write(frame)\n",
    "#     if(frameId > 7000):\n",
    "#         break;\n",
    "        \n",
    "\n",
    "    \n",
    "video.release()\n",
    "output_video.release()\n",
    "cv2.destroyAllWindows()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "conda_tensorflow_p36",
   "language": "python",
   "name": "conda_tensorflow_p36"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  },
  "toc-autonumbering": false,
  "toc-showcode": false,
  "toc-showmarkdowntxt": false
 },
 "nbformat": 4,
 "nbformat_minor": 4
}