From d75b299b8e162935e4f154a9e5d7535dd5359f21 Mon Sep 17 00:00:00 2001
From: Rusty Mina <rustymina@gmail.com>
Date: Tue, 9 Apr 2019 16:01:24 +0800
Subject: [PATCH] Fixed and added Inverted Dropout #2

---
 eureka/nn.py | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/eureka/nn.py b/eureka/nn.py
index 69b46a9..d21b0df 100644
--- a/eureka/nn.py
+++ b/eureka/nn.py
@@ -72,7 +72,7 @@ def backward(self, dh):
 
 # Dropout Layer
 class Dropout(BaseLayer):
-    def __init__(self, drop_prob):
+    def __init__(self, drop_prob, inverted = True):
         super(Dropout, self).__init__()
         # Initialize layer type
         self.layer_type = "nn.Dropout"
@@ -84,15 +84,24 @@ def __init__(self, drop_prob):
         # Initialize mask
         self.mask = None
 
+        # Inverted 
+        self.inverted = inverted
+
     def forward(self, x):
         # Generate mask
         self.mask = np.random.binomial(1, self.keep_prob, size=x.shape)
 
         # Apply mask
-        return x * self.mask
+        if (self.inverted):
+            return x * self.mask / self.keep_prob
+        else:
+            return x * self.mask
 
     def backward(self, da):
-        return da * self.mask
+        if (self.inverted):
+            return da * self.mask / self.keep_prob
+        else:
+            return da * self.mask
 
 # Normalization Layers
 class BatchNorm1d(BaseLayer):