objectClassification.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import argparse\n",
    "import os.path\n",
    "import re\n",
    "import sys\n",
    "import tarfile\n",
    "\n",
    "import numpy as np\n",
    "from six.moves import urllib\n",
    "import tensorflow as tf\n",
    "\n",
    "import glob\n",
    "import os\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import cv2\n",
    "import math\n",
    "\n",
    "DATA_URL = 'http://download.tensorflow.org/models/image/imagenet/inception-2015-12-05.tgz'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Image Recognition"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class NodeLookup(object):\n",
    "  \"\"\"Converts integer node ID's to human readable labels.\"\"\"\n",
    "\n",
    "  def __init__(self,\n",
    "               label_lookup_path=None,\n",
    "               uid_lookup_path=None):\n",
    "    if not label_lookup_path:\n",
    "        label_lookup_path = 'model/imagenet_2012_challenge_label_map_proto.pbtxt'\n",
    "    if not uid_lookup_path: \n",
    "        uid_lookup_path = 'model/imagenet_synset_to_human_label_map.txt'\n",
    "    \n",
    "    self.node_lookup = self.load(label_lookup_path, uid_lookup_path)\n",
    "\n",
    "  def load(self, label_lookup_path, uid_lookup_path):\n",
    "    \"\"\"Loads a human readable English name for each softmax node.\n",
    "\n",
    "    Args:\n",
    "      label_lookup_path: string UID to integer node ID.\n",
    "      uid_lookup_path: string UID to human-readable string.\n",
    "\n",
    "    Returns:\n",
    "      dict from integer node ID to human-readable string.\n",
    "    \"\"\"\n",
    "    if not tf.gfile.Exists(uid_lookup_path):\n",
    "      tf.logging.fatal('File does not exist %s', uid_lookup_path)\n",
    "    if not tf.gfile.Exists(label_lookup_path):\n",
    "      tf.logging.fatal('File does not exist %s', label_lookup_path)\n",
    "\n",
    "    # Loads mapping from string UID to human-readable string\n",
    "    proto_as_ascii_lines = tf.gfile.GFile(uid_lookup_path).readlines()\n",
    "    uid_to_human = {}\n",
    "    p = re.compile(r'[n\\d]*[ \\S,]*')\n",
    "    for line in proto_as_ascii_lines:\n",
    "      parsed_items = p.findall(line)\n",
    "      uid = parsed_items[0]\n",
    "      human_string = parsed_items[2]\n",
    "      uid_to_human[uid] = human_string\n",
    "\n",
    "    # Loads mapping from string UID to integer node ID.\n",
    "    node_id_to_uid = {}\n",
    "    proto_as_ascii = tf.gfile.GFile(label_lookup_path).readlines()\n",
    "    for line in proto_as_ascii:\n",
    "      if line.startswith('  target_class:'):\n",
    "        target_class = int(line.split(': ')[1])\n",
    "      if line.startswith('  target_class_string:'):\n",
    "        target_class_string = line.split(': ')[1]\n",
    "        node_id_to_uid[target_class] = target_class_string[1:-2]\n",
    "\n",
    "    # Loads the final mapping of integer node ID to human-readable string\n",
    "    node_id_to_name = {}\n",
    "    for key, val in node_id_to_uid.items():\n",
    "      if val not in uid_to_human:\n",
    "        tf.logging.fatal('Failed to locate: %s', val)\n",
    "      name = uid_to_human[val]\n",
    "      node_id_to_name[key] = name\n",
    "\n",
    "    return node_id_to_name\n",
    "\n",
    "  def id_to_string(self, node_id):\n",
    "    if node_id not in self.node_lookup:\n",
    "      return ''\n",
    "    return self.node_lookup[node_id]\n",
    "\n",
    "\n",
    "def create_graph():\n",
    "    \"\"\"Creates a graph from saved GraphDef file and returns a saver.\"\"\"\n",
    "    # Creates graph from saved graph_def.pb.\n",
    "    with tf.gfile.FastGFile('model/classify_image_graph_def.pb', 'rb') as f:\n",
    "        graph_def = tf.GraphDef()\n",
    "        graph_def.ParseFromString(f.read())\n",
    "        _ = tf.import_graph_def(graph_def, name='')\n",
    "\n",
    "def run_inference_on_image_file(images, num_top_predictions=3):\n",
    "    \"\"\"Runs inference on an image.\n",
    "\n",
    "    Args:\n",
    "    image: Image file name.\n",
    "\n",
    "    Returns:\n",
    "    Nothing\n",
    "    \"\"\"\n",
    "    for image in images :\n",
    "        if not tf.gfile.Exists(image):\n",
    "            tf.logging.fatal('File does not exist %s', image)\n",
    "    \n",
    "    images_data = {}\n",
    "    for image in images :\n",
    "        image_data = tf.gfile.FastGFile(image, 'rb').read()\n",
    "        images_data[image] = image_data\n",
    "        \n",
    "    result = {}\n",
    "    # Creates graph from saved GraphDef.\n",
    "    create_graph()\n",
    "\n",
    "    with tf.Session() as sess:\n",
    "        for filename in images_data :\n",
    "            image_data = images_data[filename]            \n",
    "            # Some useful tensors:\n",
    "            # 'softmax:0': A tensor containing the normalized prediction across\n",
    "            #   1000 labels.\n",
    "            # 'pool_3:0': A tensor containing the next-to-last layer containing 2048\n",
    "            #   float description of the image.\n",
    "            # 'DecodeJpeg/contents:0': A tensor containing a string providing JPEG\n",
    "            #   encoding of the image.\n",
    "            # Runs the softmax tensor by feeding the image_data as input to the graph.\n",
    "            softmax_tensor = sess.graph.get_tensor_by_name('softmax:0')\n",
    "            predictions = sess.run(softmax_tensor,\n",
    "                                   {'DecodeJpeg/contents:0': image_data})\n",
    "            predictions = np.squeeze(predictions)\n",
    "\n",
    "            # Creates node ID --> English string lookup.\n",
    "            node_lookup = NodeLookup()\n",
    "            file_writer = tf.summary.FileWriter('/home/izza/Project/DataExtraction', sess.graph)\n",
    "\n",
    "            prediction_summary = \"\";\n",
    "            top_k = predictions.argsort()[-num_top_predictions:][::-1]\n",
    "            for node_id in top_k:\n",
    "                human_string = node_lookup.id_to_string(node_id)\n",
    "                score = predictions[node_id]\n",
    "                prediction_summary += '%s (score = %.5f)' % (human_string, score)\n",
    "            result[filename] = prediction_summary\n",
    "    return result\n",
    "\n",
    "def run_inference_on_image(images_data, num_top_predictions=3):\n",
    "    result = {}\n",
    "    # Creates graph from saved GraphDef.\n",
    "    create_graph()\n",
    "\n",
    "    with tf.Session() as sess:\n",
    "        for filename in images_data :\n",
    "            image_data = images_data[filename]            \n",
    "            softmax_tensor = sess.graph.get_tensor_by_name('softmax:0')\n",
    "            x = tf.constant(image_data)\n",
    "            predictions = sess.run(softmax_tensor,\n",
    "                                   {'DecodeJpeg:0': image_data})\n",
    "            predictions = np.squeeze(predictions)\n",
    "\n",
    "            # Creates node ID --> English string lookup.\n",
    "            node_lookup = NodeLookup()\n",
    "#             file_writer = tf.summary.FileWriter('/home/izza/Project/DataExtraction', sess.graph)\n",
    "\n",
    "            prediction_summary = \"\";\n",
    "            top_k = predictions.argsort()[-num_top_predictions:][::-1]\n",
    "            for node_id in top_k:\n",
    "                human_string = node_lookup.id_to_string(node_id)\n",
    "                score = predictions[node_id]\n",
    "                prediction_summary += '%s (score = %.5f)' % (human_string, score)\n",
    "            result[filename] = prediction_summary\n",
    "    return result\n",
    "\n",
    "def maybe_download_and_extract():\n",
    "  \"\"\"Download and extract model tar file.\"\"\"\n",
    "  dest_directory = \"model\"\n",
    "  if not os.path.exists(dest_directory):\n",
    "    os.makedirs(dest_directory)\n",
    "  filename = DATA_URL.split('/')[-1]\n",
    "  filepath = os.path.join(dest_directory, filename)\n",
    "  if not os.path.exists(filepath):\n",
    "    def _progress(count, block_size, total_size):\n",
    "      sys.stdout.write('\\r>> Downloading %s %.1f%%' % (\n",
    "          filename, float(count * block_size) / float(total_size) * 100.0))\n",
    "      sys.stdout.flush()\n",
    "    filepath, _ = urllib.request.urlretrieve(DATA_URL, filepath, _progress)\n",
    "    print()\n",
    "    statinfo = os.stat(filepath)\n",
    "    print('Successfully downloaded', filename, statinfo.st_size, 'bytes.')\n",
    "  tarfile.open(filepath, 'r:gz').extractall(dest_directory)\n",
    "\n",
    "def recognize_object_from_file(image_files):\n",
    "  maybe_download_and_extract()\n",
    "  return run_inference_on_image_file(image_files)\n",
    "\n",
    "def recognize_object(images):\n",
    "    maybe_download_and_extract()\n",
    "    return run_inference_on_image(images)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "recognize_object_from_file([\"images/fish.png\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Video Slicing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def slice_video(videoFile = 'videos/big_buck_bunny.mp4', frameDiffTime=1, save=False, video_name=\"unknown\") : \n",
    "    cap = cv2.VideoCapture(videoFile)\n",
    "    frameRate = frameDiffTime * cap.get(5) #frame rate\n",
    "    frames = {}\n",
    "    \n",
    "    count = 0\n",
    "    while(cap.isOpened()):\n",
    "        frameId = cap.get(1) #current frame number\n",
    "        ret, frame = cap.read()\n",
    "        if (ret != True):\n",
    "            break\n",
    "        \n",
    "        if (frameId % math.floor(frameRate) == 0):\n",
    "            filename = \"videos/frames/\" +  video_name + \"_\" + str(count) + \".jpg\"\n",
    "            frames[filename] = frame\n",
    "            \n",
    "            if (save) :\n",
    "                cv2.imwrite(filename, frame)\n",
    "        count += 1\n",
    "    cap.release()\n",
    "    return frames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "frames = slice_video(\"videos/animal.mp4\", 30)\n",
    "print(\"extracted \" + str(len(frames)) + \" images\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions = recognize_object(frames)\n",
    "for key in sorted(frames):\n",
    "    plt.imshow(frames[key], cmap = 'gray', interpolation = 'bicubic')\n",
    "    plt.xticks([]), plt.yticks([])  # to hide tick values on X and Y axis\n",
    "    plt.show()\n",
    "    print(predictions[key])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prediction with Saved File"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "frames = slice_video(\"videos/big_buck_bunny.mp4\", True, \"bunny\")\n",
    "print(\"extracted \" + str(len(frames)) + \" images\")\n",
    "\n",
    "predictions = recognize_object_from_file(frames.keys())\n",
    "for key in sorted(frames):\n",
    "    plt.imshow(frames[key], cmap = 'gray', interpolation = 'bicubic')\n",
    "    plt.xticks([]), plt.yticks([])  # to hide tick values on X and Y axis\n",
    "    plt.show()\n",
    "    print(predictions[key])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}