Merge branch 'dev' into resolve_conflict

CMakeLists.txt

@@ -118,6 +118,8 @@ IF (USE_CUDA)
     #ADD_SUBDIRECTORY(lib/cnmem)
     #LIST(APPEND SINGA_LINKER_LIBS cnmem)
     SET(global_cuda_objs "")
+    # add support cuda fp16
+    SET(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} --gpu-architecture=compute_75")
 ENDIF()
 
 # TODO(wangwei) detect the ev lib

LICENSE

@@ -297,6 +297,7 @@ license and copyright terms herein.
 =====================================================================
 SINGA bundles the following under MIT license:
 cmake/ThirdParty/FindOpenCL.cmake
+include/half.hpp
 
 Copyright (c) 2010-2016 Institute for Microelectronics,
                         Institute for Analysis and Scientific Computing, TU Wien.

cmake/Dependencies.cmake

@@ -67,7 +67,7 @@ ENDIF()
 
 FIND_PACKAGE(Glog)
 IF(GLOG_FOUND)
-    #MESSAGE(STATUS "GLOG FOUND at ${GLOG_INCLUDE_DIR}")
+    MESSAGE(STATUS "FOUND GLOG at ${GLOG_INCLUDE_DIR}")
     #ADD_DEFINITIONS("-DUSE_GLOG")
     SET(USE_GLOG TRUE)
     LIST(APPEND SINGA_LINKER_LIBS ${GLOG_LIBRARIES})

examples/cnn/train_cnn.py

@@ -26,6 +26,10 @@
 import argparse
 from PIL import Image
 
+np_dtype = {"float16": np.float16, "float32": np.float32}
+
+singa_dtype = {"float16": tensor.float16, "float32": tensor.float32}
+
 
 # Data Augmentation
 def augmentation(x, batch_size):
@@ -101,7 +105,8 @@ def run(global_rank,
         graph,
         verbosity,
         dist_option='fp32',
-        spars=None):
+        spars=None,
+        precision='float32'):
     dev = device.create_cuda_gpu_on(local_rank)
     dev.SetRandSeed(0)
     np.random.seed(0)
@@ -116,6 +121,7 @@ def run(global_rank,
         from data import mnist
         train_x, train_y, val_x, val_y = mnist.load()
 
+
     num_channels = train_x.shape[1]
     image_size = train_x.shape[2]
     data_size = np.prod(train_x.shape[1:train_x.ndim]).item()
@@ -171,9 +177,9 @@ def run(global_rank,
     if model.dimension == 4:
         tx = tensor.Tensor(
             (batch_size, num_channels, model.input_size, model.input_size), dev,
-            tensor.float32)
+            singa_dtype[precision])
     elif model.dimension == 2:
-        tx = tensor.Tensor((batch_size, data_size), dev, tensor.float32)
+        tx = tensor.Tensor((batch_size, data_size), dev, singa_dtype[precision])
         np.reshape(train_x, (train_x.shape[0], -1))
         np.reshape(val_x, (val_x.shape[0], -1))
 
@@ -208,6 +214,7 @@ def run(global_rank,
                 x = augmentation(x, batch_size)
                 if (image_size != model.input_size):
                     x = resize_dataset(x, model.input_size)
+            x = x.astype(np_dtype[precision])
             y = train_y[idx[b * batch_size:(b + 1) * batch_size]]
 
             # Copy the patch data into input tensors
@@ -238,6 +245,7 @@ def run(global_rank,
             if model.dimension == 4:
                 if (image_size != model.input_size):
                     x = resize_dataset(x, model.input_size)
+            x = x.astype(np_dtype[precision])
             y = val_y[b * batch_size:(b + 1) * batch_size]
             tx.copy_from_numpy(x)
             ty.copy_from_numpy(y)
@@ -262,12 +270,17 @@ def run(global_rank,
     # use argparse to get command config: max_epoch, model, data, etc. for single gpu training
     parser = argparse.ArgumentParser(
         description='Training using the autograd and graph.')
-    parser.add_argument('model',
-                        choices=['resnet', 'xceptionnet', 'cnn', 'mlp', 'alexnet'],
-                        default='cnn')
+    parser.add_argument(
+        'model',
+        choices=['resnet', 'xceptionnet', 'cnn', 'mlp', 'alexnet'],
+        default='cnn')
     parser.add_argument('data',
                         choices=['cifar10', 'cifar100', 'mnist'],
                         default='mnist')
+    parser.add_argument('-p',
+                        choices=['float32', 'float16'],
+                        default='float32',
+                        dest='precision')
     parser.add_argument('-m',
                         '--max-epoch',
                         default=10,
@@ -308,6 +321,15 @@ def run(global_rank,
 
     args = parser.parse_args()
 
-    sgd = opt.SGD(lr=args.lr, momentum=0.9, weight_decay=1e-5)
-    run(0, 1, args.device_id, args.max_epoch, args.batch_size, args.model,
-        args.data, sgd, args.graph, args.verbosity)
+    sgd = opt.SGD(lr=args.lr, momentum=0.9, weight_decay=1e-5, dtype=singa_dtype[args.precision])
+    run(0,
+        1,
+        args.device_id,
+        args.max_epoch,
+        args.batch_size,
+        args.model,
+        args.data,
+        sgd,
+        args.graph,
+        args.verbosity,
+        precision=args.precision)

examples/mlp/model.py

@@ -20,6 +20,14 @@
 from singa import layer
 from singa import model
 from singa import tensor
+from singa import opt
+from singa import device
+import argparse
+import numpy as np
+
+np_dtype = {"float16": np.float16, "float32": np.float32}
+
+singa_dtype = {"float16": tensor.float16, "float32": tensor.float32}
 
 
 class MLP(model.Model):
@@ -78,10 +86,26 @@ def create_model(pretrained=False, **kwargs):
 __all__ = ['MLP', 'create_model']
 
 if __name__ == "__main__":
-
-    import numpy as np
-    from singa import opt
-    from singa import device
+    np.random.seed(0)
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-p',
+                        choices=['float32', 'float16'],
+                        default='float32',
+                        dest='precision')
+    parser.add_argument('-g',
+                        '--disable-graph',
+                        default='True',
+                        action='store_false',
+                        help='disable graph',
+                        dest='graph')
+    parser.add_argument('-m',
+                        '--max-epoch',
+                        default=1001,
+                        type=int,
+                        help='maximum epochs',
+                        dest='max_epoch')
+    args = parser.parse_args()
 
     # generate the boundary
     f = lambda x: (5 * x + 1)
@@ -90,22 +114,27 @@ def create_model(pretrained=False, **kwargs):
     # generate the training data
     x = np.random.uniform(-1, 1, 400)
     y = f(x) + 2 * np.random.randn(len(x))
+
+    # choose one precision
+    precision = singa_dtype[args.precision]
+    np_precision = np_dtype[args.precision]
+
     # convert training data to 2d space
     label = np.asarray([5 * a + 1 > b for (a, b) in zip(x, y)]).astype(np.int32)
-    data = np.array([[a, b] for (a, b) in zip(x, y)], dtype=np.float32)
+    data = np.array([[a, b] for (a, b) in zip(x, y)], dtype=np_precision)
 
     dev = device.create_cuda_gpu_on(0)
-    sgd = opt.SGD(0.05)
-    tx = tensor.Tensor((400, 2), dev, tensor.float32)
+    sgd = opt.SGD(0.1, 0.9, 1e-5, dtype=singa_dtype[args.precision])
+    tx = tensor.Tensor((400, 2), dev, precision)
     ty = tensor.Tensor((400,), dev, tensor.int32)
     model = MLP(data_size=2, perceptron_size=3, num_classes=2)
 
     # attached model to graph
     model.set_optimizer(sgd)
-    model.compile([tx], is_train=True, use_graph=True, sequential=False)
+    model.compile([tx], is_train=True, use_graph=args.graph, sequential=True)
     model.train()
 
-    for i in range(1001):
+    for i in range(args.max_epoch):
         tx.copy_from_numpy(data)
         ty.copy_from_numpy(label)
         out, loss = model(tx, ty, 'fp32', spars=None)

examples/mlp/native.py

@@ -22,8 +22,28 @@
 from singa import autograd
 from singa import opt
 import numpy as np
+from singa import device
+import argparse
+
+np_dtype = {"float16": np.float16, "float32": np.float32}
+
+singa_dtype = {"float16": tensor.float16, "float32": tensor.float32}
 
 if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-p',
+                        choices=['float32', 'float16'],
+                        default='float32',
+                        dest='precision')
+    parser.add_argument('-m',
+                        '--max-epoch',
+                        default=1001,
+                        type=int,
+                        help='maximum epochs',
+                        dest='max_epoch')
+    args = parser.parse_args()
+
+    np.random.seed(0)
 
     autograd.training = True
 
@@ -62,22 +82,47 @@ def to_categorical(y, num_classes):
     print("train_data_shape:", data.shape)
     print("train_label_shape:", label.shape)
 
-    inputs = Tensor(data=data)
-    target = Tensor(data=label)
+    precision = singa_dtype[args.precision]
+    np_precision = np_dtype[args.precision]
+
+    dev = device.create_cuda_gpu()
 
-    w0 = Tensor(shape=(2, 3), requires_grad=True, stores_grad=True)
-    w0.gaussian(0.0, 0.1)
-    b0 = Tensor(shape=(3,), requires_grad=True, stores_grad=True)
+    inputs = Tensor(data=data, device=dev)
+    target = Tensor(data=label, device=dev)
+
+    inputs = inputs.as_type(precision)
+    target = target.as_type(tensor.int32)
+
+    w0_np = np.random.normal(0, 0.1, (2, 3)).astype(np_precision)
+    w0 = Tensor(data=w0_np,
+                device=dev,
+                dtype=precision,
+                requires_grad=True,
+                stores_grad=True)
+    b0 = Tensor(shape=(3,),
+                device=dev,
+                dtype=precision,
+                requires_grad=True,
+                stores_grad=True)
     b0.set_value(0.0)
 
-    w1 = Tensor(shape=(3, 2), requires_grad=True, stores_grad=True)
-    w1.gaussian(0.0, 0.1)
-    b1 = Tensor(shape=(2,), requires_grad=True, stores_grad=True)
+    w1_np = np.random.normal(0, 0.1, (3, 2)).astype(np_precision)
+    w1 = Tensor(data=w1_np,
+                device=dev,
+                dtype=precision,
+                requires_grad=True,
+                stores_grad=True)
+    b1 = Tensor(shape=(2,),
+                device=dev,
+                dtype=precision,
+                requires_grad=True,
+                stores_grad=True)
     b1.set_value(0.0)
 
-    sgd = opt.SGD(0.05)
+    sgd = opt.SGD(0.05, 0.8)
+
     # training process
-    for i in range(1001):
+    for i in range(args.max_epoch):
         x = autograd.matmul(inputs, w0)
         x = autograd.add_bias(x, b0)
         x = autograd.relu(x)
@@ -87,4 +132,4 @@ def to_categorical(y, num_classes):
         sgd(loss)
 
         if i % 100 == 0:
-            print("training loss = ", tensor.to_numpy(loss)[0])
+            print("%d, training loss = " % i, tensor.to_numpy(loss)[0])