Initial implementation with LeNet-5 network architecture.

.gitignore

@@ -0,0 +1,3 @@
+simulator
+__pycache__
+data

CarND-Behavioral-Cloning-P3.iml

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager" inherit-compiler-output="true">
+    <exclude-output />
+    <content url="file://$MODULE_DIR$" />
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+</module>

LeNet.py

@@ -0,0 +1,107 @@
+from keras.models import Sequential
+from keras.layers.convolutional import Convolution2D
+from keras.layers.convolutional import MaxPooling2D
+from keras.layers.core import Activation
+from keras.layers.core import Flatten
+from keras.layers.core import Dense
+from keras.layers.core import Dropout
+from keras.layers import Lambda, Cropping2D
+
+
+class LeNet:
+    """ Standard LeNet-5 network which can be used either for a classification or regression problem."""
+
+    width = 0               # Input width
+    height = 0              # Input height
+    depth = 0               # Input depth (e.g. number of channels of an image)
+    nb_classes = 1          # Number of output classes resp. number of regression values
+    regression = False      # If true the network is setup for a regression problem. Otherwise for classification.
+    crop_top = 0            # Number of pixels the image is cropped from top row.
+    crop_bottom = 0         # Number of pixels the image is cropped from bottom row.
+    weights_path = ''       # Path to trained model weights.
+    model = None            # LeNet keras model
+
+    def __init__(self, width, height, depth, nb_classes, regression=False,
+                 crop_top=0, crop_bottom=0, weights_path=None):
+        """ Constructs the LeNet-5 network architecture.
+
+        width        -- Width of the input image.
+        height       -- Height if the input image.
+        depth        -- Depth of the input image (e.g. number of channels).
+        nb_classes   -- Number of unique classes (class labels) in the dataset. In case of a regression set, the number
+                        of regression outputs.
+        regression   -- If true the output layer is configured for a regression problem. If false the output
+                        is configured with a softmax function.
+        crop_top     -- If >0 the image will be cropped from top row by given number of pixels.
+        crop_bottom  -- If >0 the image will be cropped from bottom by given number of pixels.
+        weights_path -- Path to trained model parameters. If set, the model will be initialized by these parameters.
+        """
+
+        self.width = width
+        self.height = height
+        self.depth = depth
+        self.nb_classes = nb_classes
+        self.regression = regression
+        self.crop_top = crop_top
+        self.crop_bottom = crop_bottom
+        self.weights_path = weights_path
+
+        print('LeNet Configuration:')
+        print(' Input Layer: w={:d}, h={:d}, d={:d}'.format(self.width, self.height, self.depth))
+        print(' Output Layer: {:d}, {:s}'.format(self.nb_classes, 'regression' if self.regression else 'softmax'))
+
+        # setup-up network architecture
+        self.model = self.setup_network_architecture
+
+    @property
+    def setup_network_architecture(self):
+        """ Constructs the LeNet-5 network architecture. """
+
+        # initialize the model
+        self.model = Sequential()
+
+        # normalize and mean center images
+        self.model.add(Lambda(lambda x: x / 255.0 - 0.5, input_shape=(self.height, self.width, self.depth)))
+
+        # crop images at top and bottom
+        if self.crop_top > 0 or self.crop_bottom > 0:
+            self.model.add(Cropping2D(cropping=((self.crop_top, self.crop_bottom), (0, 0))))
+
+        # 1. layer: CONV --> POOL --> RELU
+        # 6 convolutions with 5x5 filter
+        # max pooling with 2x2 pool
+        self.model.add(Convolution2D(6, 5, 5, border_mode='valid'))
+        self.model.add(MaxPooling2D(pool_size=(2, 2)))
+        self.model.add(Activation('relu'))
+
+        # 2. layer: CONV --> POOL --> RELU
+        # 16 convolutions with 5x5 filter
+        # max pooling with 2x2 pool
+        self.model.add(Convolution2D(16, 5, 5, border_mode="valid"))
+        self.model.add(MaxPooling2D(pool_size=(2, 2)))
+        self.model.add(Activation("relu"))
+
+        # 1. fully connected layer
+        self.model.add(Flatten())
+        self.model.add(Dense(120))
+        self.model.add(Activation("relu"))
+        self.model.add(Dropout(0.5))
+
+        # 2. fully connected layer
+        self.model.add(Dense(84))
+        self.model.add(Activation("relu"))
+        self.model.add(Dropout(0.4))
+
+        # output layer
+        self.model.add(Dense(self.nb_classes))
+
+        if not self.regression:
+            # add softmax activation in case of classification setup
+            self.model.add(Activation("softmax"))
+
+        # if a weights path is supplied (indicating that the model was pre-trained), then load the weights
+        if self.weights_path is not None:
+            self.model.load_weights(self.weights_path)
+
+        # return the constructed network architecture
+        return self.model

drive.py

@@ -0,0 +1,139 @@
+import argparse
+import base64
+from datetime import datetime
+import os
+import shutil
+
+import numpy as np
+import socketio
+import eventlet
+import eventlet.wsgi
+from PIL import Image
+from flask import Flask
+from io import BytesIO
+
+from keras.models import load_model
+import h5py
+from keras import __version__ as keras_version
+
+sio = socketio.Server()
+app = Flask(__name__)
+model = None
+prev_image_array = None
+
+
+class SimplePIController:
+    def __init__(self, Kp, Ki):
+        self.Kp = Kp
+        self.Ki = Ki
+        self.set_point = 0.
+        self.error = 0.
+        self.integral = 0.
+
+    def set_desired(self, desired):
+        self.set_point = desired
+
+    def update(self, measurement):
+        # proportional error
+        self.error = self.set_point - measurement
+
+        # integral error
+        self.integral += self.error
+
+        return self.Kp * self.error + self.Ki * self.integral
+
+
+controller = SimplePIController(0.1, 0.002)
+set_speed = 9
+controller.set_desired(set_speed)
+
+
+@sio.on('telemetry')
+def telemetry(sid, data):
+    if data:
+        # The current steering angle of the car
+        steering_angle = data["steering_angle"]
+        # The current throttle of the car
+        throttle = data["throttle"]
+        # The current speed of the car
+        speed = data["speed"]
+        # The current image from the center camera of the car
+        imgString = data["image"]
+        image = Image.open(BytesIO(base64.b64decode(imgString)))
+        image_array = np.asarray(image)
+        steering_angle = float(model.predict(image_array[None, :, :, :], batch_size=1))
+
+        throttle = controller.update(float(speed))
+
+        print(steering_angle, throttle)
+        send_control(steering_angle, throttle)
+
+        # save frame
+        if args.image_folder != '':
+            timestamp = datetime.utcnow().strftime('%Y_%m_%d_%H_%M_%S_%f')[:-3]
+            image_filename = os.path.join(args.image_folder, timestamp)
+            image.save('{}.jpg'.format(image_filename))
+    else:
+        # NOTE: DON'T EDIT THIS.
+        sio.emit('manual', data={}, skip_sid=True)
+
+
+@sio.on('connect')
+def connect(sid, environ):
+    print("connect ", sid)
+    send_control(0, 0)
+
+
+def send_control(steering_angle, throttle):
+    sio.emit(
+        "steer",
+        data={
+            'steering_angle': steering_angle.__str__(),
+            'throttle': throttle.__str__()
+        },
+        skip_sid=True)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Remote Driving')
+    parser.add_argument(
+        'model',
+        type=str,
+        help='Path to model h5 file. Model should be on the same path.'
+    )
+    parser.add_argument(
+        'image_folder',
+        type=str,
+        nargs='?',
+        default='',
+        help='Path to image folder. This is where the images from the run will be saved.'
+    )
+    args = parser.parse_args()
+
+    # check that model Keras version is same as local Keras version
+    f = h5py.File(args.model, mode='r')
+    model_version = f.attrs.get('keras_version')
+    keras_version = str(keras_version).encode('utf8')
+
+    if model_version != keras_version:
+        print('You are using Keras version ', keras_version,
+              ', but the model was built using ', model_version)
+
+    model = load_model(args.model)
+
+    if args.image_folder != '':
+        print("Creating image folder at {}".format(args.image_folder))
+        if not os.path.exists(args.image_folder):
+            os.makedirs(args.image_folder)
+        else:
+            shutil.rmtree(args.image_folder)
+            os.makedirs(args.image_folder)
+        print("RECORDING THIS RUN ...")
+    else:
+        print("NOT RECORDING THIS RUN ...")
+
+    # wrap Flask application with engineio's middleware
+    app = socketio.Middleware(sio, app)
+
+    # deploy as an eventlet WSGI server
+    eventlet.wsgi.server(eventlet.listen(('', 4567)), app)