07b-cat-wrangling.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Goes with the transfer learning notebook. This is just to put the files in the right place."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# starting with train.zip in downloads\n",
    "!mv /Users/shnayder/Downloads/train.zip /Users/shnayder/data/cats_dogs/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "!cd /Users/shnayder/data/cats_dogs/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "!unzip /Users/shnayder/data/cats_dogs/train.zip"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note to self: in the future, don't dump 25K lines of output into a notebook (deleted to control file size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   25000\r\n"
     ]
    }
   ],
   "source": [
    "ls /Users/shnayder/data/cats_dogs/train|wc -l\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cat.0.jpg\r\n",
      "cat.1.jpg\r\n",
      "cat.10.jpg\r\n",
      "cat.100.jpg\r\n",
      "cat.1000.jpg\r\n",
      "cat.10000.jpg\r\n",
      "cat.10001.jpg\r\n",
      "cat.10002.jpg\r\n",
      "cat.10003.jpg\r\n",
      "cat.10004.jpg\r\n"
     ]
    }
   ],
   "source": [
    "ls /Users/shnayder/data/cats_dogs/train|head"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dog.0.jpg\r\n",
      "dog.1.jpg\r\n",
      "dog.10.jpg\r\n",
      "dog.100.jpg\r\n",
      "dog.1000.jpg\r\n",
      "dog.10000.jpg\r\n",
      "dog.10001.jpg\r\n",
      "dog.10002.jpg\r\n",
      "dog.10003.jpg\r\n",
      "dog.10004.jpg\r\n"
     ]
    }
   ],
   "source": [
    "ls /Users/shnayder/data/cats_dogs/train/|grep dog | head"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "!mkdir /Users/shnayder/data/cats_dogs/train/cats\n",
    "!mkdir /Users/shnayder/data/cats_dogs/train/dogs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Goal:\n",
    "# put the cat pictures index 0-999 in data/train/cats\n",
    "# put the cat pictures index 1000-1400 in data/validation/cats\n",
    "# put the dogs pictures index 12500-13499 in data/train/dogs\n",
    "# put the dog pictures index 13500-13900 in data/validation/dogs\n",
    "# (note: dog photos actually also named 0-12500, not 12500-25K)\n",
    "\n",
    "# really slow, but who cares...\n",
    "for i in range(1000):\n",
    "    !mv /Users/shnayder/data/cats_dogs/train/cat.{i}.jpg /Users/shnayder/data/cats_dogs/train/cats\n",
    "    !mv /Users/shnayder/data/cats_dogs/train/dog.{i}.jpg /Users/shnayder/data/cats_dogs/train/dogs\n",
    "    \n",
    "for i in range(1000,1400):\n",
    "    !mv /Users/shnayder/data/cats_dogs/train/cat.{i}.jpg /Users/shnayder/data/cats_dogs/validation/cats\n",
    "    !mv /Users/shnayder/data/cats_dogs/train/dog.{i}.jpg /Users/shnayder/data/cats_dogs/validation/dogs    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/Users/shnayder/data/cats_dogs/train/cats: 1000\n",
      "/Users/shnayder/data/cats_dogs/train/dogs: 1000\n",
      "/Users/shnayder/data/cats_dogs/validation/cats: 400\n",
      "/Users/shnayder/data/cats_dogs/validation/dogs: 400\n"
     ]
    }
   ],
   "source": [
    "# double check\n",
    "data_root = os.path.expanduser(\"~/data/cats_dogs\")\n",
    "train_data_dir = os.path.join(data_root, 'train')\n",
    "validation_data_dir = os.path.join(data_root, 'validation')\n",
    "\n",
    "# Make sure we have the expected numbers of images\n",
    "for d in [train_data_dir, validation_data_dir]:\n",
    "    for category in ['cats', 'dogs']:\n",
    "        print(\"{}/{}: {}\".format(\n",
    "                d, category, len(os.listdir(os.path.join(d, category)))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Clean up the other files so they don't clog my hard drive\n",
    "for name in os.listdir(train_data_dir):\n",
    "    if name.endswith('.jpg'):\n",
    "        filepath = os.path.join(train_data_dir, name)\n",
    "        os.remove(filepath)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Oops. Forgot to rename to dog001/cat001, etc. Let's do that."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for i in range(0, 1000):\n",
    "    old = os.path.join(train_data_dir, 'cats', 'cat.'+str(i)+'.jpg')\n",
    "    new = os.path.join(train_data_dir, 'cats', 'cat{:03d}.jpg'.format(i))\n",
    "    os.rename(old, new)\n",
    "\n",
    "    old = os.path.join(train_data_dir, 'dogs', 'dog.'+str(i)+'.jpg')\n",
    "    new = os.path.join(train_data_dir, 'dogs', 'dog{:03d}.jpg'.format(i))\n",
    "    os.rename(old, new)\n",
    "    \n",
    "for i in range(1000, 1400):\n",
    "    old = os.path.join(validation_data_dir, 'cats', 'cat.'+str(i)+'.jpg')\n",
    "    new = os.path.join(validation_data_dir, 'cats', 'cat{:03d}.jpg'.format(i-1000))\n",
    "    os.rename(old, new)\n",
    "\n",
    "    old = os.path.join(validation_data_dir, 'dogs', 'dog.'+str(i)+'.jpg')\n",
    "    new = os.path.join(validation_data_dir, 'dogs', 'dog{:03d}.jpg'.format(i-1000))\n",
    "    os.rename(old, new)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cat000.jpg\r\n",
      "cat001.jpg\r\n",
      "cat002.jpg\r\n",
      "cat003.jpg\r\n",
      "cat004.jpg\r\n",
      "cat005.jpg\r\n",
      "cat006.jpg\r\n",
      "cat007.jpg\r\n",
      "cat008.jpg\r\n",
      "cat009.jpg\r\n"
     ]
    }
   ],
   "source": [
    "!ls {train_data_dir}/cats | head"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dog000.jpg\r\n",
      "dog001.jpg\r\n",
      "dog002.jpg\r\n",
      "dog003.jpg\r\n",
      "dog004.jpg\r\n",
      "dog005.jpg\r\n",
      "dog006.jpg\r\n",
      "dog007.jpg\r\n",
      "dog008.jpg\r\n",
      "dog009.jpg\r\n"
     ]
    }
   ],
   "source": [
    "!ls {train_data_dir}/dogs | head"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dog000.jpg\r\n",
      "dog001.jpg\r\n",
      "dog002.jpg\r\n",
      "dog003.jpg\r\n",
      "dog004.jpg\r\n",
      "dog005.jpg\r\n",
      "dog006.jpg\r\n",
      "dog007.jpg\r\n",
      "dog008.jpg\r\n",
      "dog009.jpg\r\n"
     ]
    }
   ],
   "source": [
    "!ls {validation_data_dir}/dogs | head"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Phew. Looks ok now."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}