Added Linear Support Vector Machine Classifier

Examples/datasets/svm_classification.txt

@@ -0,0 +1,40 @@
+-7.609293811334917 2.757350307161618 1
+3.634038257043118 14.215834756980865 1
+-5.191746876631818 4.794905134719064 1
+9.056466416958227 20.89387291134578 1
+-5.674886856985648 4.06850020569974 1
+-7.412163011788763 2.040290062355872 1
+-5.437522194913131 5.873366744168288 1
+5.268552651922754 14.626563039736551 1
+-0.9144714432731327 8.853174840513711 1
+4.800803793218204 16.56934921242772 1
+-4.8848181913029265 4.976425895523652 1
+1.3673385039428432 10.404181024201405 1
+0.4535171875310091 8.848399270206821 1
+9.488215089090556 20.509330776408984 1
+-6.786273881581857 2.203258444507078 1
+-8.94829021637885 0.9631174111646914 1
+-8.896223728582699 0.4445654168242469 1
+-0.24459310585761607 9.527781049072129 1
+-5.169562669961804 6.367759979934328 1
+9.456696217028007 19.616738464977328 1
+-6.735807418424686 12.925468675219518 -1
+7.573009500222913 28.359761293776177 -1
+9.765453905888986 29.816895339007075 -1
+-8.127099589780368 11.063036796733938 -1
+-5.577306800134892 14.133446660743399 -1
+1.7409301686303653 23.282335179913773 -1
+8.276378892557826 27.73254308204849 -1
+-1.4041002656961066 18.41936918910657 -1
+7.054949347896233 26.309404118090892 -1
+-5.037948865890298 15.532280213196367 -1
+5.9827876786470675 25.168179875339376 -1
+-6.303995121064969 11.755175823299137 -1
+4.695616256767446 24.0674751175545 -1
+5.7583446641596545 26.223326524706817 -1
+-8.287207639346544 12.57734135918184 -1
+1.6284581281435173 22.001572361690556 -1
+0.988746941665573 21.108757597915695 -1
+4.647281318057619 25.21924375417958 -1
+-6.2226152784368445 12.87341228150541 -1
+8.242042235112276 26.82751524587767 -1

Examples/svm_classification_example.py

@@ -0,0 +1,7 @@
+from MLlib.models import LinearSVC
+import numpy as np
+A = np.genfromtxt("datasets/svm_classification.txt")
+X = A[:, 0:-1]
+Y = A[:, -1].flatten()
+clf = LinearSVC()
+clf.plot(X, Y, epochs=1000)

MLlib/loss_func.py

@@ -31,7 +31,7 @@ def forward(ctx, prediction, target):
         if requires_grad:
             ctx.derivative_core = out
 
-        out = np.sum(np.power(out, 2)) / (2*batch_size)
+        out = np.sum(np.power(out, 2)) / (2 * batch_size)
 
         output = Tensor(out, requires_grad=requires_grad,
                         is_leaf=not requires_grad)
@@ -125,7 +125,7 @@ def loss(X, Y, W):
         M = X.shape[0]
         sigmoid = Sigmoid()
         H = sigmoid.activation(np.dot(X, W).T)
-        return (1/M)*(np.sum((-Y)*np.log(H)-(1-Y)*np.log(1-H)))
+        return (1 / M) * (np.sum((-Y) * np.log(H) - (1 - Y) * np.log(1 - H)))
 
     @staticmethod
     def derivative(X, Y, W):
@@ -150,7 +150,7 @@ def derivative(X, Y, W):
         M = X.shape[0]
         sigmoid = Sigmoid()
         H = sigmoid.activation(np.dot(X, W).T)
-        return (1/M)*(np.dot(X.T, (H-Y).T))
+        return (1 / M) * (np.dot(X.T, (H - Y).T))
 
 
 class AbsoluteError():
@@ -202,15 +202,15 @@ def derivative(X, Y, W):
          array of derivates
         """
         M = X.shape[0]
-        AbsError = (np.dot(X, W).T-Y)
+        AbsError = (np.dot(X, W).T - Y)
         return np.dot(
             np.divide(
                 AbsError,
                 np.absolute(AbsError),
                 out=np.zeros_like(AbsError),
                 where=(np.absolute(AbsError)) != 0),
             X
-        ).T/M
+        ).T / M
 
 
 class CosineSimilarity():
@@ -240,9 +240,10 @@ def loss(X, Y, W):
          """
         H = (np.dot(X, W).T)
         DP = np.sum(np.dot(H, Y))
-        S = DP/((np.sum(np.square(H))**(0.5))*(np.sum(np.square(Y))**(0.5)))
-        dissimilarity = 1-S
-        return dissimilarity*(np.sum(np.square(Y))**(0.5))
+        S = DP / ((np.sum(np.square(H))**(0.5))
+                  * (np.sum(np.square(Y))**(0.5)))
+        dissimilarity = 1 - S
+        return dissimilarity * (np.sum(np.square(Y))**(0.5))
 
 
 class Log_cosh():
@@ -292,7 +293,7 @@ def derivative_logcosh(X, Y, W):
 
         Derivative of Log cosh prediction error
         """
-        t = np.tanh(Y-np.dot(X, W).T) @ (-X)
+        t = np.tanh(Y - np.dot(X, W).T) @ (-X)
         derivative = np.sum(t)
         return derivative
 
@@ -304,7 +305,6 @@ class Huber():
 
     @staticmethod
     def loss(X, Y, W, delta=1.0):
-
         """
         Calculate loss by Huber method.
 
@@ -327,12 +327,11 @@ def loss(X, Y, W, delta=1.0):
         M = X.shape[0]
         error = np.where(np.abs(Y - y_pred) <= delta,
                          0.5 * (Y - y_pred)**2,
-                         delta * (np.abs(Y - y_pred)-0.5*delta))
+                         delta * (np.abs(Y - y_pred) - 0.5 * delta))
         return np.sum(error) / M
 
     @staticmethod
     def derivative(X, Y, W, delta=1.0):
-
         """
         Calculate derivative for Huber method.
 
@@ -425,3 +424,79 @@ def loss(X, Y, W):
         y_pred = np.dot(X, W).T
         L = np.sum(np.true_divide((np.abs(Y - y_pred) * 100), Y)) / X.shape[0]
         return L
+
+
+class SVMLoss():
+    """
+    Calculates the loss function involved in Support Vector Machine model.
+    """
+    reg_strength = 1
+
+    def __init__(self, reg_strength=1):
+        self.reg_strength = reg_strength
+
+    def loss(self, X, Y, W):
+        """
+            Calculates the loss function
+            involved in Support Vector Machine Model.
+
+            PARAMETERS
+            ==========
+
+            X:ndarray(dtype=float,ndim=1)
+              input vector
+            Y:ndarray(dtype=float)
+              output vector
+            W:ndarray(dtype=float)
+              Weights
+
+            RETURNS
+            =======
+            The value of loss function
+            involved in support vector machine model.
+        """
+        if isinstance(Y, np.float64):
+            Y = np.array([Y])
+            X = np.array([X])
+        W = W.flatten()
+        W0 = W[:-1]
+        X = np.hstack((X, np.ones((X.shape[0], 1))))
+        distances = 1 - Y * (np.dot(X, W))
+        distances[distances < 0] = 0
+        L = (self.reg_strength / 2) * np.dot(W0, W0) + np.sum(distances)
+        return L
+
+    def derivative(self, X, Y, W):
+        """
+        Calculate derivative for SVM loss function.
+
+        PARAMETERS
+        ==========
+
+        X:ndarray(dtype=float,ndim=1)
+          input vector
+        Y:ndarray(dtype=float)
+          output vector
+        W:ndarray(dtype=float)
+          Weights
+
+         RETURNS
+         =======
+
+         array of derivates
+        """
+        if isinstance(Y, np.int64):
+            Y = np.array([Y])
+            X = np.array([X])
+        W = W.flatten()
+        X = np.hstack((X, np.ones((X.shape[0], 1))))
+        first_term_derivative = self.reg_strength * np.append(W[:-1], [0])
+        second_term_derivative = np.zeros((X.shape[1],))
+        distances = 1 - Y * (np.dot(X, W))
+        distances[distances < 0] = 0
+        for i, d in enumerate(distances):
+            if d > 0:
+                second_term_derivative += Y[i] * X[i]
+        D = first_term_derivative - (second_term_derivative / len(Y))
+        D.resize((D.shape[0], 1))
+        return D