add manifold learning

Author: johmathe

Diff for: examples/mnist_cnn.py

@@ -12,6 +12,7 @@
 from keras.layers import Dense, Dropout, Flatten
 from keras.layers import Conv2D, MaxPooling2D
 from keras import backend as K
+from geomstats.hypersphere import Hypersphere
 
 batch_size = 128
 num_classes = 10
@@ -47,8 +48,9 @@
 model = Sequential()
 model.add(Conv2D(32, kernel_size=(3, 3),
                  activation='relu',
-                 input_shape=input_shape))
-model.add(Conv2D(64, (3, 3), activation='relu'))
+                 input_shape=input_shape,
+                 kernel_manifold=Hypersphere(dimension=32*3*3)))
+model.add(Conv2D(64, (3, 3), activation='relu', kernel_manifold=Hypersphere(dimension=64*3*3)))
 model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(Dropout(0.25))
 model.add(Flatten())
@@ -57,7 +59,7 @@
 model.add(Dense(num_classes, activation='softmax'))
 
 model.compile(loss=keras.losses.categorical_crossentropy,
-              optimizer=keras.optimizers.Adadelta(),
+              optimizer=keras.optimizers.SGD(),
               metrics=['accuracy'])
 
 model.fit(x_train, y_train,

Diff for: keras/backend/tensorflow_backend.py

@@ -356,7 +356,7 @@ def to_dense(tensor):
 name_scope = tf.name_scope
 
 
-def variable(value, dtype=None, name=None, constraint=None):
+def variable(value, dtype=None, name=None, constraint=None, manifold=None):
     """Instantiates a variable and returns it.
 
     # Arguments
@@ -406,6 +406,8 @@ def variable(value, dtype=None, name=None, constraint=None):
         v.constraint = constraint
     except AttributeError:
         v._constraint = constraint
+
+    v._manifold = manifold
     return v
 
 

Diff for: keras/engine/base_layer.py

@@ -220,7 +220,8 @@ def add_weight(self,
                    initializer=None,
                    regularizer=None,
                    trainable=True,
-                   constraint=None):
+                   constraint=None,
+                   manifold=None):
         """Adds a weight variable to the layer.
 
         # Arguments
@@ -243,7 +244,8 @@ def add_weight(self,
         weight = K.variable(initializer(shape),
                             dtype=dtype,
                             name=name,
-                            constraint=constraint)
+                            constraint=constraint,
+                            manifold=manifold)
         if regularizer is not None:
             with K.name_scope('weight_regularizer'):
                 self.add_loss(regularizer(weight))

Diff for: keras/layers/convolutional.py

@@ -99,6 +99,7 @@ def __init__(self, rank,
                  bias_regularizer=None,
                  activity_regularizer=None,
                  kernel_constraint=None,
+                 kernel_manifold=None,
                  bias_constraint=None,
                  **kwargs):
         super(_Conv, self).__init__(**kwargs)
@@ -114,6 +115,7 @@ def __init__(self, rank,
         self.kernel_initializer = initializers.get(kernel_initializer)
         self.bias_initializer = initializers.get(bias_initializer)
         self.kernel_regularizer = regularizers.get(kernel_regularizer)
+        self.kernel_manifold = kernel_manifold
         self.bias_regularizer = regularizers.get(bias_regularizer)
         self.activity_regularizer = regularizers.get(activity_regularizer)
         self.kernel_constraint = constraints.get(kernel_constraint)
@@ -134,6 +136,7 @@ def build(self, input_shape):
         self.kernel = self.add_weight(shape=kernel_shape,
                                       initializer=self.kernel_initializer,
                                       name='kernel',
+                                      manifold=self.kernel_manifold,
                                       regularizer=self.kernel_regularizer,
                                       constraint=self.kernel_constraint)
         if self.use_bias:
@@ -443,6 +446,7 @@ def __init__(self, filters,
                  bias_regularizer=None,
                  activity_regularizer=None,
                  kernel_constraint=None,
+                 kernel_manifold=None,
                  bias_constraint=None,
                  **kwargs):
         super(Conv2D, self).__init__(
@@ -461,9 +465,11 @@ def __init__(self, filters,
             bias_regularizer=bias_regularizer,
             activity_regularizer=activity_regularizer,
             kernel_constraint=kernel_constraint,
+            kernel_manifold=kernel_manifold,
             bias_constraint=bias_constraint,
             **kwargs)
         self.input_spec = InputSpec(ndim=4)
+        print(kernel_manifold)
 
     def get_config(self):
         config = super(Conv2D, self).get_config()

Diff for: keras/layers/core.py

@@ -835,6 +835,7 @@ def __init__(self, units,
                  bias_regularizer=None,
                  activity_regularizer=None,
                  kernel_constraint=None,
+                 kernel_manifold=None,
                  bias_constraint=None,
                  **kwargs):
         if 'input_shape' not in kwargs and 'input_dim' in kwargs:
@@ -849,6 +850,7 @@ def __init__(self, units,
         self.bias_regularizer = regularizers.get(bias_regularizer)
         self.activity_regularizer = regularizers.get(activity_regularizer)
         self.kernel_constraint = constraints.get(kernel_constraint)
+        self.kernel_manifold = kernel_manifold
         self.bias_constraint = constraints.get(bias_constraint)
         self.input_spec = InputSpec(min_ndim=2)
         self.supports_masking = True
@@ -860,6 +862,7 @@ def build(self, input_shape):
         self.kernel = self.add_weight(shape=(input_dim, self.units),
                                       initializer=self.kernel_initializer,
                                       name='kernel',
+                                      manifold=self.kernel_manifold,
                                       regularizer=self.kernel_regularizer,
                                       constraint=self.kernel_constraint)
         if self.use_bias:

Diff for: keras/optimizers.py

@@ -1,5 +1,4 @@
-"""Built-in optimizer classes.
-"""
+
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
@@ -178,15 +177,26 @@ def get_updates(self, loss, params):
             lr *= (1. / (1. + self.decay * K.cast(self.iterations,
                                                   K.dtype(self.decay))))
         # momentum
+        # TODO(johmathe): Add back nesterov.
         shapes = [K.int_shape(p) for p in params]
         moments = [K.zeros(shape) for shape in shapes]
         self.weights = [self.iterations] + moments
         for p, g, m in zip(params, grads, moments):
-            v = self.momentum * m - lr * g  # velocity
+            v = self.momentum * m - lr * g
             self.updates.append(K.update(m, v))
 
-            if self.nesterov:
-                new_p = p + self.momentum * v - lr * g
+            # Do the gradient descent on the manifold if present.
+            if getattr(p, 'manifold', None) is not None:
+                print('MANIF')
+                shape = K.shape(v)
+                manifold = p.manifold
+                v_shaped = K.reshape(v, (manifold.dimension, -1))
+                p_shaped = K.reshape(p, (manifold.dimension, -1))
+                tangent_v = manifold.projection_to_tangent_space(
+                        vector=v_shaped, base_point=p_shaped)
+                destination = manifold.metric.exp(
+                        base_point=p_shaped, vector=tangent_v)
+                new_p = K.reshape(destination, shape, name='new_p')
             else:
                 new_p = p + v
 
@@ -253,6 +263,7 @@ def get_updates(self, loss, params):
 
         for p, g, a in zip(params, grads, accumulators):
             # update accumulator
+
             new_a = self.rho * a + (1. - self.rho) * K.square(g)
             self.updates.append(K.update(a, new_a))
             new_p = p - lr * g / (K.sqrt(new_a) + self.epsilon)