Advanced_Lane_Finding_P4.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Project: Advanced Lane Finding\n",
    "---\n",
    "### The goals / steps of this project are the following:\n",
    "\n",
    "* Compute the camera calibration matrix and distortion coefficients given a set of chessboard images.\n",
    "* Apply a distortion correction to raw images.\n",
    "* Use color transforms, gradients, etc., to create a thresholded binary image.\n",
    "* Apply a perspective transform to rectify binary image (\"birds-eye view\").\n",
    "* Detect lane pixels and fit to find the lane boundary.\n",
    "* Determine the curvature of the lane and vehicle position with respect to center.\n",
    "* Warp the detected lane boundaries back onto the original image.\n",
    "* Output visual display of the lane boundaries and numerical estimation of lane curvature and vehicle position.\n",
    "---"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import Packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Importing some useful packages\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.image as mpimg\n",
    "import numpy as np\n",
    "import cv2\n",
    "import glob\n",
    "from time import sleep\n",
    "%matplotlib inline\n",
    "\n",
    "# Global debug flags\n",
    "DEBUG = False\n",
    "DEBUG_2 = False"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Camera Calibration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Prepare object points\n",
    "nx = 9 # number of inside corners in x\n",
    "ny = 6 # number of inside corners in y\n",
    "\n",
    "objpoints = []\n",
    "imgpoints = []\n",
    "\n",
    "# Prepare object points, like (0,0,0), (1,0,0), (2,0,0) ..., (nx-1,ny-1,0)\n",
    "objp = np.zeros((nx*ny,3), np.float32)\n",
    "objp[:,:2] = np.mgrid[0:nx,0:ny].T.reshape(-1,2) # x, y coordinates\n",
    "\n",
    "# Make a list of calibration images\n",
    "fnames = glob.glob('camera_cal/calibration*.jpg')\n",
    "\n",
    "# Iterate trough all calibration images\n",
    "for fname in fnames:\n",
    "    img = mpimg.imread(fname)\n",
    "    \n",
    "    # Convert to grayscale\n",
    "    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)\n",
    "\n",
    "    # Find the chessboard corners\n",
    "    ret, corners = cv2.findChessboardCorners(gray, (nx, ny), None)\n",
    "\n",
    "    # If found, add object points and image points\n",
    "    if ret == True:\n",
    "        imgpoints.append(corners)\n",
    "        objpoints.append(objp)\n",
    "        if DEBUG:\n",
    "            # Draw and display the corners\n",
    "            cv2.drawChessboardCorners(img, (nx, ny), corners, ret)\n",
    "            plt.imshow(img)\n",
    "            plt.show()\n",
    "    else:\n",
    "        if DEBUG: print(\"No corners found in the following image:\", fname)\n",
    "\n",
    "# Camera calibration\n",
    "ret, mtx, dist, rvecs, tvecs = cv2.calibrateCamera(objpoints, imgpoints, gray.shape[:2], None, None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Test camera calibration on one of the calibration images\n",
    "\n",
    "f, (ax1, ax2) = plt.subplots(1, 2, figsize=(15,4))\n",
    "\n",
    "img = mpimg.imread(fnames[0])\n",
    "ax1.imshow(img)\n",
    "ax1.set_title('Test Image', fontsize=20)\n",
    "\n",
    "undist = cv2.undistort(img, mtx, dist, None, mtx)\n",
    "ax2.set_title('Undistorted Test Image', fontsize=20)\n",
    "ax2.imshow(undist)\n",
    "\n",
    "plt.savefig('grid_figure.png')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Helper Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Definition of some helper functions and classes that will be used later\n",
    "\n",
    "def undistort(img, mtx, dist):\n",
    "    \"\"\"\n",
    "    This function receives a distorted image and returns it undistorted.\n",
    "    Args:\n",
    "        img: image to be undistorted\n",
    "        mtx: camera matrix\n",
    "        dist: distortion coefficients\n",
    "    Returns:\n",
    "        imgage: undistorted version of the input image\n",
    "    \"\"\"\n",
    "    return cv2.undistort(img, mtx, dist, None, mtx)\n",
    "\n",
    "def lane_filter(img, h_thresh=(15, 40), s_thresh=(256, 255), sx_thresh=(50, 255), white_thresh = 198):\n",
    "    \"\"\"\n",
    "    This function receives a road image with lane lines and returns a binary image\n",
    "    where the lane lines are marked with 1s and most of the rest is marked with 0s.\n",
    "    \n",
    "    Args:\n",
    "        img (RGB-image): road image with lane lines\n",
    "        h_thresh (int,int): min and max hue-value for HLS-image filter\n",
    "        s_stesh (int,int): min and max saturation-value for HLS-image filter\n",
    "        sx_thresh (int,int): min and max sobel-x-value\n",
    "        white_thresh (int): min value for all RGB-color channels to find white pixels\n",
    "    \n",
    "    Returns:\n",
    "        binary image: the lane lines are marked with 1s and most of the rest is marked with 0s\n",
    "    \"\"\"\n",
    "    img = np.copy(img)\n",
    "    \n",
    "    r_channel = img[:,:,0]\n",
    "    g_channel = img[:,:,1]\n",
    "    b_channel = img[:,:,2]\n",
    "    \n",
    "    # White threshold\n",
    "    w_binary = np.zeros_like(r_channel)\n",
    "    w_binary[(r_channel >= white_thresh) & (g_channel >= white_thresh) & (b_channel >= white_thresh)] = 1\n",
    "    \n",
    "    # Convert to HLS color space and separate the channels\n",
    "    hls = cv2.cvtColor(img, cv2.COLOR_RGB2HLS).astype(np.float)\n",
    "    h_channel = hls[:,:,0]\n",
    "    l_channel = hls[:,:,1]\n",
    "    s_channel = hls[:,:,2]\n",
    "    r_minus_l_channel = r_channel - l_channel\n",
    "    \n",
    "    # Threshold color channels\n",
    "    s_binary = np.zeros_like(s_channel)\n",
    "    s_binary[(s_channel >= s_thresh[0]) & (s_channel <= s_thresh[1])] = 1\n",
    "    \n",
    "    hs_binary = np.zeros_like(h_channel)\n",
    "    hs_binary[(h_channel >= h_thresh[0]) & (h_channel <= h_thresh[1]) & (s_channel >= 90)] = 1\n",
    "    \n",
    "    \n",
    "    # Sobel x\n",
    "    sobelx = cv2.Sobel(r_minus_l_channel, cv2.CV_64F, 1, 0, ksize=3) # Take the derivative in x of the `r_minus_l_channel`\n",
    "    abs_sobelx = np.absolute(sobelx) # Absolute x derivative to accentuate lines away from horizontal\n",
    "    scaled_sobel = np.uint8(255*abs_sobelx/np.max(abs_sobelx))\n",
    "    \n",
    "    # Threshold x gradient\n",
    "    sx_binary = np.zeros_like(scaled_sobel)\n",
    "    sx_binary[(scaled_sobel >= sx_thresh[0]) & (scaled_sobel <= sx_thresh[1]) & \\\n",
    "              (s_channel >= 12) & (l_channel >= 12)] = 1\n",
    "    \n",
    "    # For Debugging Only: Stack each channel\n",
    "    # Note color_binary[:, :, 0] is all 0s, effectively an all black image. It might\n",
    "    # be beneficial to replace this channel with something else.\n",
    "#     color_binary = np.dstack((w_binary, s_binary, sx_binary))\n",
    "#     return color_binary\n",
    "\n",
    "    combined = np.zeros_like(s_binary)\n",
    "    combined[(hs_binary == 1) | (s_binary == 1) | (w_binary == 1) | (sx_binary == 1)] = 1\n",
    "    return combined\n",
    "\n",
    "def fit_lines(binary_warped):\n",
    "    \"\"\"\n",
    "    Fit two lane lines in a binary image using sliding window search.\n",
    "    \n",
    "    Args:\n",
    "        binary_warped (binary image): warped binary image (bird's eye view),\n",
    "            where ideally lane pixels are '1' and most other pixels are '0'\n",
    "    Returns:\n",
    "        left_fit, right_fit: second order polynomial coefficients that define the detected lane lines\n",
    "    \"\"\"\n",
    "    # Assuming you have created a warped binary image called \"binary_warped\"\n",
    "    # Take a histogram of the bottom half of the image\n",
    "    histogram = np.sum(binary_warped[binary_warped.shape[0]//2:,:], axis=0)\n",
    "    # Create an output image to draw on and  visualize the result\n",
    "    out_img = np.dstack((binary_warped, binary_warped, binary_warped))*255\n",
    "    # Find the peak of the left and right halves of the histogram\n",
    "    # These will be the starting point for the left and right lines\n",
    "    \n",
    "    midpoint = np.int(histogram.shape[0]/2)\n",
    "    leftx_base = np.argmax(histogram[:midpoint])\n",
    "    rightx_base = np.argmax(histogram[midpoint:]) + midpoint\n",
    "    # To be precise: the histogram is cut into seven parts\n",
    "    # and we search only part 2 and 3 for the left starting point\n",
    "    # and we search only part 5 and 6 for the right starting point:\n",
    "    # 7 parts of the histogram:  |x|L|L|x|R|R|x|  (x: not searched, L/R: search for left/right starting point)\n",
    "#     one7th = np.int(histogram.shape[0]/7.)\n",
    "#     leftx_base = np.argmax(histogram[one7th : (3*one7th)]) + one7th\n",
    "#     rightx_base = np.argmax(histogram[(4*one7th) : (6*one7th)]) + 4*one7th\n",
    "    \n",
    "    # Choose the number of sliding windows\n",
    "    nwindows = 9\n",
    "    # Set height of windows\n",
    "    window_height = np.int(binary_warped.shape[0]/nwindows)\n",
    "    # Identify the x and y positions of all nonzero pixels in the image\n",
    "    nonzero = binary_warped.nonzero()\n",
    "    nonzeroy = np.array(nonzero[0])\n",
    "    nonzerox = np.array(nonzero[1])\n",
    "    # Current positions to be updated for each window\n",
    "    leftx_current = leftx_base\n",
    "    rightx_current = rightx_base\n",
    "    # Set the width of the windows +/- margin\n",
    "    margin = 100\n",
    "    # Set minimum number of pixels found to recenter window\n",
    "    minpix = 50\n",
    "    # Create empty lists to receive left and right lane pixel indices\n",
    "    left_lane_inds = []\n",
    "    right_lane_inds = []\n",
    "\n",
    "    # Step through the windows one by one\n",
    "    for window in range(nwindows):\n",
    "        # Identify window boundaries in x and y (and right and left)\n",
    "        win_y_low = binary_warped.shape[0] - (window+1)*window_height\n",
    "        win_y_high = binary_warped.shape[0] - window*window_height\n",
    "        win_xleft_low = leftx_current - margin\n",
    "        win_xleft_high = leftx_current + margin\n",
    "        win_xright_low = rightx_current - margin\n",
    "        win_xright_high = rightx_current + margin\n",
    "        # Draw the windows on the visualization image\n",
    "        cv2.rectangle(out_img,(win_xleft_low,win_y_low),(win_xleft_high,win_y_high),(0,255,0), 2) \n",
    "        cv2.rectangle(out_img,(win_xright_low,win_y_low),(win_xright_high,win_y_high),(0,255,0), 2) \n",
    "        # Identify the nonzero pixels in x and y within the window\n",
    "        good_left_inds = ((nonzeroy >= win_y_low) & (nonzeroy < win_y_high) & (nonzerox >= win_xleft_low) & (nonzerox < win_xleft_high)).nonzero()[0]\n",
    "        good_right_inds = ((nonzeroy >= win_y_low) & (nonzeroy < win_y_high) & (nonzerox >= win_xright_low) & (nonzerox < win_xright_high)).nonzero()[0]\n",
    "        # Append these indices to the lists\n",
    "        left_lane_inds.append(good_left_inds)\n",
    "        right_lane_inds.append(good_right_inds)\n",
    "        # If you found > minpix pixels, recenter next window on their mean position\n",
    "        if len(good_left_inds) > minpix:\n",
    "            leftx_current = np.int(np.mean(nonzerox[good_left_inds]))\n",
    "        if len(good_right_inds) > minpix:        \n",
    "            rightx_current = np.int(np.mean(nonzerox[good_right_inds]))\n",
    "\n",
    "    # Concatenate the arrays of indices\n",
    "    left_lane_inds = np.concatenate(left_lane_inds)\n",
    "    right_lane_inds = np.concatenate(right_lane_inds)\n",
    "\n",
    "    # Extract left and right line pixel positions\n",
    "    leftx = nonzerox[left_lane_inds]\n",
    "    lefty = nonzeroy[left_lane_inds] \n",
    "    rightx = nonzerox[right_lane_inds]\n",
    "    righty = nonzeroy[right_lane_inds] \n",
    "\n",
    "    # Fit a second order polynomial to each\n",
    "    if leftx.any():\n",
    "        left_fit = np.polyfit(lefty, leftx, 2)\n",
    "    else:\n",
    "        left_fit = []\n",
    "    if rightx.any():\n",
    "        right_fit = np.polyfit(righty, rightx, 2)\n",
    "    else:\n",
    "        right_fit = [] \n",
    "\n",
    "    return left_fit, right_fit\n",
    "\n",
    "def fit_lines_fast(binary_warped, left_fit, right_fit):\n",
    "    \"\"\"\n",
    "    Fit two lane lines in a binary image based on previously detected lines.\n",
    "    \n",
    "    Args:\n",
    "        binary_warped (binary image): warped binary image (bird's eye view),\n",
    "            where ideally lane pixels are '1' and most other pixels are '0'\n",
    "        left_fit (list): second order polynomial coefficients that define the last known left line\n",
    "        right_fit (list): second order polynomial coefficients that define the last known right line\n",
    "    Returns:\n",
    "        left_fit, right_fit: second order polynomial coefficients that define the detected lane lines\n",
    "    \"\"\"\n",
    "    nonzero = binary_warped.nonzero()\n",
    "    nonzeroy = np.array(nonzero[0])\n",
    "    nonzerox = np.array(nonzero[1])\n",
    "    margin = 100\n",
    "    left_lane_inds = ((nonzerox > (left_fit[0]*(nonzeroy**2) + left_fit[1]*nonzeroy + left_fit[2] - margin)) & (nonzerox < (left_fit[0]*(nonzeroy**2) + left_fit[1]*nonzeroy + left_fit[2] + margin))) \n",
    "    right_lane_inds = ((nonzerox > (right_fit[0]*(nonzeroy**2) + right_fit[1]*nonzeroy + right_fit[2] - margin)) & (nonzerox < (right_fit[0]*(nonzeroy**2) + right_fit[1]*nonzeroy + right_fit[2] + margin)))  \n",
    "\n",
    "    # Again, extract left and right line pixel positions\n",
    "    leftx = nonzerox[left_lane_inds]\n",
    "    lefty = nonzeroy[left_lane_inds] \n",
    "    rightx = nonzerox[right_lane_inds]\n",
    "    righty = nonzeroy[right_lane_inds]\n",
    "    # Fit a second order polynomial to each\n",
    "    left_fit = np.polyfit(lefty, leftx, 2)\n",
    "    right_fit = np.polyfit(righty, rightx, 2)\n",
    "    \n",
    "    return left_fit, right_fit\n",
    "\n",
    "def region_of_interest(img, vertices, inside=True):\n",
    "    \"\"\"\n",
    "    Applies an image mask. For binary images only!\n",
    "    \n",
    "    Only keeps the region of the image defined by the polygon\n",
    "    formed from `vertices`. The rest of the image is set to black (for inside=True).\n",
    "    Or vice versa if parameter \"inside\" is set to False.\n",
    "    \"\"\"\n",
    "    # depending on the value of 'inside', define a all-zeros or all-ones mask to start with\n",
    "    if inside:\n",
    "        mask = np.zeros_like(img)\n",
    "        mask_color = 1\n",
    "    else:\n",
    "        mask = np.ones_like(img)\n",
    "        mask_color = 0\n",
    "        \n",
    "    #filling pixels inside the polygon defined by \"vertices\" with the fill color    \n",
    "    cv2.fillPoly(mask, vertices, mask_color)\n",
    "    \n",
    "    #returning the image only where mask pixels are nonzero\n",
    "    masked_image = cv2.bitwise_and(img, mask)\n",
    "    return masked_image\n",
    "\n",
    "\n",
    "def weighted_img(img, initial_img, α=0.8, β=1., λ=0.):\n",
    "    \"\"\"\n",
    "    `img` is the output of the hough_lines(), An image with lines drawn on it.\n",
    "    Should be a blank image (all black) with lines drawn on it.\n",
    "    \n",
    "    `initial_img` should be the image before any processing.\n",
    "    \n",
    "    The result image is computed as follows:\n",
    "    \n",
    "    initial_img * α + img * β + λ\n",
    "    NOTE: initial_img and img must be the same shape!\n",
    "    \"\"\"\n",
    "    return cv2.addWeighted(initial_img, α, img, β, λ)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## The Line and Lane classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class Line():\n",
    "    \"\"\"\n",
    "    A class to receive the characteristics of each line detection.\n",
    "    This class also defines a couple of methods to handel this data.\n",
    "    \"\"\"\n",
    "    def __init__(self, lane, position, hist_size=4, failed_max=5):\n",
    "        # lane object which holds this line and the other line\n",
    "        self.lane = lane\n",
    "        # lane position, e.g. 'left' or 'right'\n",
    "        self.position = position\n",
    "        # number of recent fits that will be stored\n",
    "        self.hist_size = hist_size\n",
    "        # maximum number of failed detections in a row before the search history\n",
    "        # will be deleted and the search will be started from scratch\n",
    "        self.failed_max = failed_max\n",
    "        \n",
    "        # history of the last n fits...\n",
    "        \n",
    "        # was the line detected in the last iteration?\n",
    "        self.detected = False\n",
    "        # number of failed line detections in a row\n",
    "        self.failed_count = 0\n",
    "        # radius of curvature of the last n fits of the line in meters\n",
    "        self.recent_radius = []\n",
    "        # average radius of curvature of the last n fits of the line in meters\n",
    "        self.best_radius = None\n",
    "        # the mean x values of the last n fits of the line\n",
    "        self.recent_x_mean = []\n",
    "        # average mean x values of the fitted line over the last n iterations\n",
    "        self.best_x_mean = None\n",
    "        # polynomial coefficients of the last n fits of the line\n",
    "        self.recent_fits = []\n",
    "        # polynomial coefficients averaged over the last n iterations\n",
    "        self.best_fit = None\n",
    "        \n",
    "        # values of current fit...\n",
    "        \n",
    "        # polynomial coefficients for the most recent fit\n",
    "        self.current_fit = [np.array([False])]\n",
    "        # x values for detected line pixels\n",
    "        self.allx = None\n",
    "        # y values for detected line pixels\n",
    "        self.ally = None\n",
    "        # radius of curvature of the line in meters\n",
    "        self.radius_of_curvature = None\n",
    "        # distance in pixels of vehicle center from the line\n",
    "        self.line_base_pos = None\n",
    "    \n",
    "    def calc_values_for_current_fit(self, current_fit):\n",
    "        \"\"\"\n",
    "        Calculate all values for the current_fit, like x- and y-values for the lane line and its radius of curvature.\n",
    "        \n",
    "        Args:\n",
    "            current_fit (list): polynomial coefficients for the most recent line fit\n",
    "        Returns:\n",
    "            None\n",
    "        \"\"\"\n",
    "        self.current_fit = current_fit\n",
    "        \n",
    "        # Calculate the x and y values of the lane for the current fit\n",
    "        self.ally = np.linspace(0, self.lane.img_size_y-1, self.lane.img_size_y)\n",
    "        self.allx = current_fit[0]*self.ally**2 + current_fit[1]*self.ally + current_fit[2]\n",
    "        \n",
    "        # Calculate the radius of curvature in meters...\n",
    "        \n",
    "        # Define y-value where we want radius of curvature\n",
    "        # I'll choose the maximum y-value, corresponding to the bottom of the image\n",
    "        y_eval = int(np.max(self.ally))\n",
    "\n",
    "        # Define conversions in x and y from pixels space to meters\n",
    "        ym_per_pix = 30/720 # meters per pixel in y dimension\n",
    "        xm_per_pix = 3.7/700 # meters per pixel in x dimension\n",
    "\n",
    "        # Fit new polynomials to x,y in world space\n",
    "        fit_cr = np.polyfit(self.ally*ym_per_pix, self.allx*xm_per_pix, 2)\n",
    "        # Calculate the new radius of curvature\n",
    "        self.radius_of_curvature = ((1 + (2*fit_cr[0]*y_eval*ym_per_pix + fit_cr[1])**2)**1.5) / np.absolute(2*fit_cr[0])\n",
    "        \n",
    "        # Calculate the distance in pixels of the vehicle center from the line\n",
    "        # the center of the image in x direction is equivalent to the center of the vehicle\n",
    "        self.line_base_pos = self.allx[y_eval] - (self.lane.img_size_x / 2.)\n",
    "        \n",
    "    def validate_current_fit(self):\n",
    "        \"\"\"\n",
    "        Validate the current fit and if possible compare it to the history of detected lines.\n",
    "        \n",
    "        Args:\n",
    "            -\n",
    "        Returns:\n",
    "            validation result (bool): True if the validation was successful, False if not.\n",
    "        \n",
    "        \"\"\"\n",
    "        if self.position == 'left':\n",
    "            if self.line_base_pos > 0:\n",
    "                if DEBUG: print('Bad line_base_pos for left line!')\n",
    "                return False\n",
    "        if self.position == 'right':\n",
    "            if self.line_base_pos < 0:\n",
    "                if DEBUG: print('Bad line_base_pos for right line!')\n",
    "                return False\n",
    "        \n",
    "        if self.radius_of_curvature < 200.0:\n",
    "            if DEBUG: print('Bad radius_of_curvature for {:} line!'.format(self.position))\n",
    "            return False\n",
    "        \n",
    "        # Do history based checks, if we have a history with at least 2 entries\n",
    "        if len(self.recent_fits) > 1:\n",
    "            # We only check their ratio if both the current line and the best averaged line are not kind of straight\n",
    "            kinda_straight_tresh = 2000.0\n",
    "            if self.radius_of_curvature < kinda_straight_tresh and self.best_radius < kinda_straight_tresh:\n",
    "                max_ratio = 5.\n",
    "                min_ratio = 1./max_ratio\n",
    "                if abs(self.radius_of_curvature / self.best_radius) < min_ratio:\n",
    "                    if DEBUG: print('{:} radius_of_curvature is too small compared to average radius!'.format(self.position))\n",
    "                    return False\n",
    "                if abs(self.radius_of_curvature / self.best_radius) > max_ratio:\n",
    "                    if DEBUG: print('{:} radius_of_curvature is too big compared to average radius!'.format(self.position))\n",
    "                    return False\n",
    "            # Check average x-value of this Line\n",
    "            current_average_x = sum(self.allx)/len(self.allx)\n",
    "            max_x_diff = 50\n",
    "            if current_average_x > self.best_x_mean + max_x_diff:\n",
    "                if DEBUG: print('{:} average x-value={:.2f} is too big compared to best_x_mean={:.2f}!' \\\n",
    "                                .format(self.position, current_average_x, self.best_x_mean))\n",
    "                return False\n",
    "            if current_average_x < self.best_x_mean - max_x_diff:\n",
    "                if DEBUG: print('{:} average x-value={:.2f} is too small compared to best_x_mean={:.2f}!' \\\n",
    "                                .format(self.position, current_average_x, self.best_x_mean))\n",
    "                return False\n",
    "            \n",
    "        return True\n",
    "    \n",
    "    def update_history(self, detected=False):\n",
    "        \"\"\"\n",
    "        Updates the history of this Line object.\n",
    "        When the line was detected (and validated) successfully (detected=True),\n",
    "        the recent fit will be appended to the list of recent_fits and all kinds of average values will be calculated.\n",
    "        If no line was detected or the detected line was not validated successfully (detected=False),\n",
    "        the history attributes will not be changed, except for the 'failed_count' which will be incremented.\n",
    "        \n",
    "        Args:\n",
    "            detected (bool): Should be True when the last line detection was successful, False otherwise.\n",
    "        Returns:\n",
    "            None\n",
    "        \"\"\"\n",
    "        if DEBUG_2: print('Updating history for {:} line: detected={:}, self.failed_count={:}, self.failed_max={:}' \\\n",
    "                       .format(self.position, detected, self.failed_count, self.failed_max))\n",
    "        \n",
    "        if detected:\n",
    "            self.detected = True\n",
    "            self.failed_count = 0 # reset failed_count to 0\n",
    "            \n",
    "            # Remove oldest element(s), when list size is equal or bigger than hist_size\n",
    "            if len(self.recent_radius) >= self.hist_size:\n",
    "                self.recent_radius = self.recent_radius[-(self.hist_size-1):]\n",
    "            # Append newest radius_of_curvature\n",
    "            self.recent_radius.append(self.radius_of_curvature)\n",
    "            # Calculate new mean\n",
    "            self.best_radius = sum(self.recent_radius)/len(self.recent_radius)\n",
    "            \n",
    "            # Remove oldest element(s), when list size is equal or bigger than hist_size\n",
    "            if len(self.recent_x_mean) >= self.hist_size:\n",
    "                self.recent_x_mean = self.recent_x_mean[-(self.hist_size-1):]\n",
    "            # Append newest x_mean-element (mean of self.allx)\n",
    "            self.recent_x_mean.append(sum(self.allx)/len(self.allx))\n",
    "            # Calculate new mean\n",
    "            self.best_x_mean = sum(self.recent_x_mean)/len(self.recent_x_mean)\n",
    "            \n",
    "            # Remove oldest element(s), when list size is equal or bigger than hist_size\n",
    "            if len(self.recent_fits) >= self.hist_size:\n",
    "                self.recent_fits = self.recent_fits[-(self.hist_size-1):]\n",
    "            # Append newest polynomial coefficients\n",
    "            self.recent_fits.append(self.current_fit)\n",
    "            # Calculate new mean for each of the polynomial coefficients\n",
    "            self.best_fit = [sum(i)/len(self.recent_fits) for i in zip(*self.recent_fits)]\n",
    "            \n",
    "        else:\n",
    "            self.detected = False\n",
    "            self.failed_count += 1\n",
    "            if self.failed_count > self.failed_max:\n",
    "                #failed_count overflow --> reset history of this line object\n",
    "                self.__init__(self.lane, self.position, hist_size=self.hist_size, failed_max=self.failed_max)\n",
    "                if DEBUG: print('History of {:} line cleared. Because failed_max = {:} has been exceeded'.format(self.position, self.failed_max))\n",
    "    \n",
    "    def calculate_average_fit(self):\n",
    "        \"\"\"\n",
    "        Calculate all values for the current fit like 'radius_of_curvature' using self.best_fit.\n",
    "        When self.best_fit is None, do nothing and return None.\n",
    "        \n",
    "        Args:\n",
    "            -\n",
    "        Returns:\n",
    "            best_fit (list): polynomial coefficients for the best averaged line fit\n",
    "        \"\"\"\n",
    "        if self.best_fit is not None:\n",
    "            self.calc_values_for_current_fit(self.best_fit)\n",
    "        return self.best_fit\n",
    "\n",
    "class Lane():\n",
    "    \"\"\"\n",
    "    The Lane class holds two lane lines as objects of the class Line.\n",
    "    \"\"\"\n",
    "    def __init__(self, img_size_x, img_size_y, lane_width=3.7, hist_size=3, failed_max=6):\n",
    "        # x size of the lane image\n",
    "        self.img_size_x = img_size_x\n",
    "        # y size of the lane image\n",
    "        self.img_size_y = img_size_y\n",
    "        # Lane width in meters\n",
    "        self.lane_width = lane_width\n",
    "        \n",
    "        # Create two Line objects\n",
    "        self.left_line = Line(self, 'left', hist_size, failed_max)\n",
    "        self.right_line = Line(self, 'right', hist_size, failed_max)\n",
    "        \n",
    "        # Vehicle position relative to the lane center in meters\n",
    "        self.vehicle_position = None\n",
    "    \n",
    "    def calc_all_values_for_current_fit(self, left_fit, right_fit):\n",
    "        \"\"\"\n",
    "        Calculates all values for the detected left and right lane line and also the current vehicle position.\n",
    "        \n",
    "        Args:\n",
    "            left_fit (list): polynomial coefficients for the most recent left line fit\n",
    "            right_fit (list): polynomial coefficients for the most recent right line fit\n",
    "        Returns:\n",
    "            None\n",
    "        \"\"\"\n",
    "        self.left_line.calc_values_for_current_fit(left_fit)\n",
    "        self.right_line.calc_values_for_current_fit(right_fit)\n",
    "        self.calc_vehicle_position()\n",
    "        \n",
    "    def calc_vehicle_position(self):\n",
    "        \"\"\"\n",
    "        Calculate vehicle position relative to middle the two lines\n",
    "        \n",
    "        Args: -\n",
    "        Returns: None\n",
    "        \"\"\"\n",
    "        # First calculate the distance of the two lane lines in pixels...\n",
    "        line_dist_px = self.right_line.line_base_pos - self.left_line.line_base_pos\n",
    "        # ...so we can calculate the 'pixels per meter' value\n",
    "        px_per_meter = line_dist_px / self.lane_width\n",
    "        \n",
    "        mid_of_vehicle_px = img_size_x / 2.\n",
    "        vehicle_pos_px = self.left_line.line_base_pos + (line_dist_px / 2.)\n",
    "        self.vehicle_position = vehicle_pos_px / px_per_meter\n",
    "        \n",
    "        \n",
    "    def validate_curvature_ratio(self, max_curvature_ratio=75.0):\n",
    "        \"\"\"\n",
    "        Carry out a sanity check on the current rato of curvature of the two lines, if the lines are not quasi straight\n",
    "        The line history will not be evaluated.\n",
    "        \n",
    "        Args:\n",
    "            max_curvature_ratio (float): maximum ratio of curvature of the two lane lines\n",
    "        Returns:\n",
    "            check result (bool): True for success, False otherwise\n",
    "        \"\"\"\n",
    "        \n",
    "        # If both lines are kind of straight, we don't check the ratio\n",
    "        kinda_straight_tresh = 1000.0\n",
    "        if self.left_line.radius_of_curvature > kinda_straight_tresh and self.right_line.radius_of_curvature > kinda_straight_tresh:\n",
    "            return True\n",
    "        \n",
    "        min_curvature_ratio = 1./max_curvature_ratio\n",
    "        curvature_ratio = self.left_line.radius_of_curvature / self.right_line.radius_of_curvature\n",
    "        if curvature_ratio <= max_curvature_ratio and curvature_ratio >= min_curvature_ratio:\n",
    "            return True\n",
    "        else:\n",
    "            if DEBUG: print('curvature_ratio={:} is not between min={:} and max={:}'.format(curvature_ratio, min_curvature_ratio, max_curvature_ratio))\n",
    "            return False"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Advanced Lane Detection Pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pipeline(image):\n",
    "    \"\"\"\n",
    "    Advanced lane detection pipeline.\n",
    "    \n",
    "    Args:\n",
    "        image (rgb-image): \n",
    "        left_line (Line): an object of the Line class which holds information about the recently detected left lane lines\n",
    "        right_line (Line): same for the right lane line\n",
    "    Returns:\n",
    "        rgb-image: undistorted version of the original image including highlighting of the detected lane\n",
    "    \"\"\"\n",
    "    # Make a copy of the input image, so we don't change the original\n",
    "    img = image.copy()\n",
    "    # Copy the Lane object which must have been defined outside of this function\n",
    "    lane = my_lane\n",
    "    # Make local copies of the image's x and y size\n",
    "    img_size_x, img_size_y = lane.img_size_x, lane.img_size_y\n",
    "    \n",
    "    ### Apply a distortion correction to raw images. ################################################\n",
    "    undist_img = undistort(img, mtx, dist)\n",
    "    \n",
    "    \n",
    "    ### Use color transforms, gradients, etc., to create a thresholded binary image. ################\n",
    "    filtered_img = lane_filter(undist_img)\n",
    "    \n",
    "    \n",
    "    # Define a four sided polygon to mask the image\n",
    "    imshape = filtered_img.shape\n",
    "    poly_height = 0.40  # height of the polygon in proportion of the image height\n",
    "    poly_bottom_width = 0.90  # in proportion of image width\n",
    "    poly_top_width = 0.10  # in proportion of image width\n",
    "    poly_x_offset = 0.02  # offset of the polygon in x direction in proportion of the image width\n",
    "    vertices = np.array([[\\\n",
    "        (((1+poly_top_width)/2 + poly_x_offset) * imshape[1], (1-poly_height)*imshape[0]),\\\n",
    "        (((1-poly_top_width)/2 + poly_x_offset) * imshape[1], (1-poly_height)*imshape[0]),\\\n",
    "        (((1-poly_bottom_width)/2 + poly_x_offset) * imshape[1], imshape[0]),\\\n",
    "        (((1+poly_bottom_width)/2 + poly_x_offset) * imshape[1], imshape[0])]], dtype=np.int32)\n",
    "    # The outside area of the polygon will be dismissed\n",
    "    masked_filtered_img = region_of_interest(filtered_img, vertices)\n",
    "    \n",
    "    # Define a three sided polygon (triangle) to mask the image\n",
    "    poly_height = 0.3\n",
    "    poly_bottom_width = 0.45\n",
    "    poly_x_offset = 0.02  # offset of the polygon in x direction in proportion of the image width\n",
    "    vertices = np.array([[\\\n",
    "        ((0.5 + poly_x_offset) * img_size_x, (1-poly_height)*img_size_y),\\\n",
    "        (((1-poly_bottom_width)/2 + poly_x_offset) * img_size_x, img_size_y),\\\n",
    "        (((1+poly_bottom_width)/2 + poly_x_offset) * img_size_x, img_size_y)]], dtype=np.int32)\n",
    "    # The outside area of the triangle will be left as it is, the inside will be dismissed\n",
    "    masked_filtered_img = region_of_interest(masked_filtered_img, vertices, inside=False)\n",
    "    \n",
    "    \n",
    "    ### Apply a perspective transform to rectify binary image (\"birds-eye view\"). ###################\n",
    "    dst_x_offset = 300\n",
    "#     src = np.float32([[img_size_x/2 - 56, 456],[img_size_x/2 + 56, 456], \\\n",
    "#                      [img_size_x/2 + 420, 694],[img_size_x/2 - 420, 694]])\n",
    "    src = np.float32([[img_size_x/2 - 56, 456],[img_size_x/2 + 56, 456], \\\n",
    "                     [img_size_x/2 + 446, 719],[img_size_x/2 - 446, 719]])\n",
    "    # c) define 4 destination points dst = np.float32([[,],[,],[,],[,]])\n",
    "    dst = np.float32([[img_size_x/2 - dst_x_offset, 0],[img_size_x/2 + dst_x_offset, 0], \\\n",
    "                       [img_size_x/2 + dst_x_offset, 719],[img_size_x/2 - dst_x_offset, 719]])\n",
    "    # d) use cv2.getPerspectiveTransform() to get M, the transform matrix\n",
    "    M = cv2.getPerspectiveTransform(src, dst)\n",
    "    Minv = cv2.getPerspectiveTransform(dst, src)\n",
    "    # e) use cv2.warpPerspective() to warp your image to a top-down view\n",
    "    binary_warped = cv2.warpPerspective(masked_filtered_img, M, (img_size_x, img_size_y), flags=cv2.INTER_LINEAR)\n",
    "\n",
    "    \n",
    "    ### Detect lane pixels and fit to find the lane boundary. #######################################\n",
    "    ### Determine the curvature of the lane and vehicle position with respect to center. ############\n",
    "    \n",
    "    if lane.left_line.detected and lane.right_line.detected:\n",
    "        # If we have detected both lines last time, we run a quick search\n",
    "        left_fit, right_fit = fit_lines_fast(binary_warped, lane.left_line.best_fit, lane.right_line.best_fit)\n",
    "    else:\n",
    "        # If we have not detected BOTH last time, we run a complete new serch (sliding window method)\n",
    "        if DEBUG: print('Using fit_lines (\"slow\")')\n",
    "        left_fit, right_fit = fit_lines(binary_warped)\n",
    "    \n",
    "    left_fit_status = 0\n",
    "    right_fit_status = 0\n",
    "    \n",
    "    if left_fit == []: # we didn't find a left line\n",
    "        if lane.left_line.recent_fits == []: # ...and we have no lines in the hisory?\n",
    "            left_fit_status = 0 # we don't have any value for the left line\n",
    "            if DEBUG: print(\"left_fit_status = 0\")\n",
    "        else:\n",
    "            left_fit_status = 1 # we use an old average value for the left line\n",
    "            if DEBUG: print(\"left_fit_status = 1\")\n",
    "            left_fit = lane.left_line.best_fit\n",
    "    else:\n",
    "        left_fit_status = 2 # all good\n",
    "    \n",
    "    if right_fit == []: # we didn't find a right line\n",
    "        if lane.right_line.recent_fits == []: # ...and we have no lines in the hisory?\n",
    "            right_fit_status = 0 # we don't have any value for the right line\n",
    "            if DEBUG: print(\"right_fit_status = 0\")\n",
    "        else:\n",
    "            right_fit_status = 1 # we use an old average value for the left line\n",
    "            if DEBUG: print(\"right_fit_status = 1\")\n",
    "            right_fit = lane.right_line.best_fit\n",
    "    else:\n",
    "        right_fit_status = 2 # all good\n",
    "\n",
    "    if left_fit_status and right_fit_status: # we have a fit for both lines\n",
    "        # Calculate all required values for the current fit like lane pixels, curvature and vehicle position\n",
    "        lane.calc_all_values_for_current_fit(left_fit, right_fit)\n",
    "    elif left_fit_status: # we have a fit only for the left line\n",
    "        lane.left_line.calc_values_for_current_fit(left_fit)\n",
    "    elif right_fit_status: # we have a fit only for the right line\n",
    "        lane.right_line.calc_values_for_current_fit(right_fit)\n",
    "    \n",
    "    \n",
    "    # Do sanity checks...\n",
    "    \n",
    "    # Initialize check flags\n",
    "    left_line_good, right_line_good, curvature_ratio_good = False, False, False\n",
    "    \n",
    "    # 1. Check individual lines\n",
    "    # a) check left line if we have a left_fit\n",
    "    if left_fit_status == 2: # we have a current left fit\n",
    "        left_line_good = lane.left_line.validate_current_fit()\n",
    "        if (not left_line_good) and (lane.left_line.recent_fits != []):\n",
    "            # the current fit is bad but we can use the 'best fit' from the history\n",
    "            left_fit = lane.left_line.best_fit\n",
    "            lane.left_line.calc_values_for_current_fit(left_fit)\n",
    "            left_line_good = True\n",
    "            left_fit_status = 1 # we're using a value from history\n",
    "            if right_fit_status:\n",
    "                # we have updated our left_fit, so we need to recalculate the vehicle position if we also have a right fit\n",
    "                lane.calc_vehicle_position()\n",
    "    \n",
    "    # b) check right line if we have a right_fit\n",
    "    if right_fit_status == 2:  # we have a current right fit\n",
    "        right_line_good = lane.right_line.validate_current_fit()\n",
    "        if (not right_line_good) and (lane.right_line.recent_fits != []):\n",
    "            # the current fit is bad but we can use the 'best fit' from the history\n",
    "            right_fit = lane.right_line.best_fit\n",
    "            lane.right_line.calc_values_for_current_fit(right_fit)\n",
    "            right_line_good = True\n",
    "            right_fit_status = 1 # we're using a value from history\n",
    "            if left_fit_status:\n",
    "                # we have updated our right_fit, so we need to recalculate the vehicle position if we also have a left fit\n",
    "                lane.calc_vehicle_position()\n",
    "    \n",
    "    # 2. Check the curvature ratio if the individual lines are good\n",
    "    if left_line_good and right_line_good:\n",
    "        curvature_ratio_good = lane.validate_curvature_ratio()\n",
    "    \n",
    "    \n",
    "    # Update history and calculate new best_fit\n",
    "    if left_line_good and (left_fit_status == 2):\n",
    "        lane.left_line.update_history(True)\n",
    "    else:\n",
    "        lane.left_line.update_history(False)\n",
    "        \n",
    "    if right_line_good and (right_fit_status == 2):\n",
    "        lane.right_line.update_history(True)\n",
    "    else:\n",
    "        lane.right_line.update_history(False)\n",
    "    \n",
    "    # recalculate all necesary values with the new best fit (if we have one)\n",
    "    left_fit = lane.left_line.calculate_average_fit()\n",
    "    right_fit = lane.right_line.calculate_average_fit()\n",
    "    if (left_fit is not None) and (right_fit is not None) and curvature_ratio_good:\n",
    "        lane.calc_vehicle_position()\n",
    "    else:\n",
    "        # if one of the lines is missing, we just return the undistorted image\n",
    "        if DEBUG: print('NO LANE OUTPUT!')\n",
    "        return undist_img\n",
    "    \n",
    "    ploty = lane.left_line.ally\n",
    "    left_fitx = lane.left_line.allx\n",
    "    right_fitx = lane.right_line.allx\n",
    "    \n",
    "#     #print the lane lines to an image...\n",
    "#     lane_points = np.column_stack((ploty, left_fitx, right_fitx))\n",
    "#     wraped_lane_img = np.copy(image)*0\n",
    "#     wraped_lane_img[binary_warped == 1] = [255,255,255]\n",
    "#     for lane_point in lane_points:\n",
    "#         if lane_point[1] < img_size_x and lane_point[1] >= 0:\n",
    "#             wraped_lane_img[int(lane_point[0]), int(lane_point[1])] = [255,0,0]\n",
    "#             wraped_lane_img[int(lane_point[0]), int(lane_point[1])+1] = [255,0,0]\n",
    "#             wraped_lane_img[int(lane_point[0]), int(lane_point[1])-1] = [255,0,0]\n",
    "#         if lane_point[2] < img_size_x and lane_point[2] >= 0:\n",
    "#             wraped_lane_img[int(lane_point[0]), int(lane_point[2])] = [0,255,0]\n",
    "#             wraped_lane_img[int(lane_point[0]), int(lane_point[2])+1] = [0,255,0]\n",
    "#             wraped_lane_img[int(lane_point[0]), int(lane_point[2])-1] = [0,255,0]\n",
    "\n",
    "    \n",
    "    ### Warp the detected lane boundaries back onto the original image and output the lanes. ########\n",
    "    # Create an image to draw the lines on\n",
    "    warp_zero = np.zeros_like(binary_warped).astype(np.uint8)\n",
    "    color_warp = np.dstack((warp_zero, warp_zero, warp_zero))\n",
    "\n",
    "    # Recast the x and y points into usable format for cv2.fillPoly()\n",
    "    pts_left = np.array([np.transpose(np.vstack([left_fitx, ploty]))])\n",
    "    pts_right = np.array([np.flipud(np.transpose(np.vstack([right_fitx, ploty])))])\n",
    "    pts = np.hstack((pts_left, pts_right))\n",
    "\n",
    "    # Draw the lane onto the warped blank image\n",
    "    cv2.fillPoly(color_warp, np.int_([pts]), (0,255, 0))\n",
    "    #cv2.fillPoly(color_warp, np.array([[0,0],[200,0],[200,200],[0,200]], np.int32), (0,255, 0))\n",
    "    \n",
    "    # Warp the blank back to original image space using inverse perspective matrix (Minv)\n",
    "    newwarp = cv2.warpPerspective(color_warp, Minv, (image.shape[1], image.shape[0])) \n",
    "    # Combine the result with the original image\n",
    "    result = cv2.addWeighted(undist_img, 1, newwarp, 0.3, 0)\n",
    "    \n",
    "    \n",
    "    #### Output numerical estimation of lane curvature and vehicle position. ########################\n",
    "    \n",
    "    font = cv2.FONT_HERSHEY_SIMPLEX\n",
    "    # Output lane curvature radius\n",
    "    text = 'Radius of curvature: {:.0f} m'.format((lane.left_line.best_radius + lane.right_line.best_radius) / 2.)\n",
    "    cv2.putText(result,text,(10,40), font, 1,(255,255,255),2,cv2.LINE_AA)\n",
    "    if DEBUG:\n",
    "        text = 'left_curverad: ' + str(lane.left_line.best_radius) + \\\n",
    "                'm, right_curverad: ' + str(lane.right_line.best_radius) + 'm'\n",
    "        cv2.putText(result,text,(10,120), font, 1,(255,255,255),2,cv2.LINE_AA)\n",
    "    \n",
    "    # See if we are left or right of center\n",
    "    if lane.vehicle_position > 0:\n",
    "        left_right = 'left'\n",
    "    else:\n",
    "        left_right = 'right'\n",
    "    \n",
    "    # Output vehicle position\n",
    "    text = \"Vehicle is {:.2f} m {} of center.\".format(abs(lane.vehicle_position), left_right)\n",
    "    cv2.putText(result,text,(10,80), font, 1,(255,255,255),2,cv2.LINE_AA)\n",
    "    \n",
    "    return result"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Apply The Pipeline To Test Images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Make a list of test images\n",
    "fnames = glob.glob('test_images/*.jpg')\n",
    "\n",
    "# Iterate trough all test images and apply the pipleline defined above\n",
    "\n",
    "for filename in fnames:\n",
    "    img = mpimg.imread(filename)\n",
    "    \n",
    "    #Get image shape\n",
    "    img_size_y, img_size_x = img.shape[:2]\n",
    "    #Create a Lane object for that image shape\n",
    "    my_lane = Lane(img_size_x, img_size_y)\n",
    "    \n",
    "    #Run the lane detection pipeline\n",
    "    result_img = pipeline(img)\n",
    "    \n",
    "    # Plot the resulting image next to the original image\n",
    "    f, (ax1, ax2) = plt.subplots(1, 2, figsize=(24, 9))\n",
    "    f.tight_layout()\n",
    "    ax1.imshow(img)\n",
    "    ax1.set_title('Original Image', fontsize=30)\n",
    "    ax2.imshow(result_img)\n",
    "    ax2.set_title('Pipeline Result', fontsize=30)\n",
    "    plt.subplots_adjust(left=0., right=1, top=0.9, bottom=0.)\n",
    "\n",
    "    plt.show()\n",
    "    \n",
    "    # Save image\n",
    "    mpimg.imsave(\"output_images/\" + filename.split('\\\\')[-1], result_img)\n",
    "    \n",
    "    sleep(0.01)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Apply The Pipeline To Videos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Import everything needed to edit/save/watch video clips\n",
    "from moviepy.editor import VideoFileClip\n",
    "from IPython.display import HTML"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### project_video.mp4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "output_file1 = 'output_videos/project_video_out.mp4'\n",
    "\n",
    "# clip1 = VideoFileClip(\"project_video.mp4\").subclip(35,45.5) #use a subclip for debugging .subclip(start_second,end_second)\n",
    "clip1 = VideoFileClip(\"project_video.mp4\")\n",
    "\n",
    "#Get clip shape\n",
    "img_size_x, img_size_y = clip1.size\n",
    "#Create a Lane object for that image shape\n",
    "my_lane = Lane(img_size_x, img_size_y)\n",
    "\n",
    "clip1 = clip1.fl_image(pipeline) #NOTE: this function expects color images!!\n",
    "%time clip1.write_videofile(output_file1, audio=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "HTML(\"\"\"\n",
    "<video width=\"960\" height=\"540\" controls>\n",
    "  <source src=\"{0}\">\n",
    "</video>\n",
    "\"\"\".format(output_file1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### challenge_video.mp4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "output_file2 = 'output_videos/challenge_video_out.mp4'\n",
    "\n",
    "#clip2 = VideoFileClip(\"challenge_video.mp4\").subclip(0,5.5) #use a subclip for debugging .subclip(start_second,end_second)\n",
    "clip2 = VideoFileClip(\"challenge_video.mp4\")\n",
    "\n",
    "#Get clip shape\n",
    "img_size_x, img_size_y = clip2.size\n",
    "#Create a Lane object for that image shape\n",
    "my_lane = Lane(img_size_x, img_size_y)\n",
    "\n",
    "clip2 = clip2.fl_image(pipeline) #NOTE: this function expects color images!!\n",
    "%time clip2.write_videofile(output_file2, audio=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "HTML(\"\"\"\n",
    "<video width=\"960\" height=\"540\" controls>\n",
    "  <source src=\"{0}\">\n",
    "</video>\n",
    "\"\"\".format(output_file2))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  },
  "widgets": {
   "state": {},
   "version": "1.1.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}