New NN classes

extra/nn/L1Cost.lua                       : L1 penalty
extra/nn/SpatialFullConvolution.lua       : full convolution
extra/nn/SpatialFullConvolutionMap.lua    : full convolution with connection table
extra/nn/TanhShrink.lua                   : shrinkage with x-tanh(x)
extra/nn/WeightedMSECriterion.lua         : mean squared error with weighting mask on the target

Add new nn classes that are used commonly for unsupervised training of convolutional auto encoders

Author: koraykv

L1Cost.lua

@@ -0,0 +1,14 @@
+local L1Cost, parent = torch.class('nn.L1Cost','nn.Criterion')
+
+function L1Cost:__init()
+   parent.__init(self)
+end
+
+function L1Cost:updateOutput(input)
+   return input.nn.L1Cost_updateOutput(self,input)
+end
+
+function L1Cost:updateGradInput(input)
+   return input.nn.L1Cost_updateGradInput(self,input)
+end
+

SpatialFullConvolution.lua

@@ -0,0 +1,53 @@
+local SpatialFullConvolution, parent = torch.class('nn.SpatialFullConvolution','nn.Module')
+
+function SpatialFullConvolution:__init(nInputPlane, nOutputPlane, kW, kH, dW, dH)
+   parent.__init(self)
+
+   dW = dW or 1
+   dH = dH or 1
+
+   self.nInputPlane = nInputPlane
+   self.nOutputPlane = nOutputPlane
+   self.kW = kW
+   self.kH = kH
+   self.dW = dW
+   self.dH = dH
+
+   self.weight = torch.Tensor(nInputPlane, nOutputPlane, kH, kW)
+   self.gradWeight = torch.Tensor(nInputPlane, nOutputPlane, kH, kW)
+   self.bias = torch.Tensor(self.nOutputPlane)
+   self.gradBias = torch.Tensor(self.nOutputPlane)
+
+   self:reset()
+end
+
+function SpatialFullConvolution:reset(stdv)
+   if stdv then
+      stdv = stdv * math.sqrt(3)
+   else
+      local nInputPlane = self.nInputPlane
+      local kH = self.kH
+      local kW = self.kW
+      stdv = 1/math.sqrt(kW*kH*nInputPlane)
+   end
+   self.weight:apply(function()
+                        return torch.uniform(-stdv, stdv)
+                     end)
+   self.bias:apply(function()
+                        return torch.uniform(-stdv, stdv)
+                     end)
+end
+
+function SpatialFullConvolution:updateOutput(input)
+   return input.nn.SpatialFullConvolution_updateOutput(self, input)
+end
+
+function SpatialFullConvolution:updateGradInput(input, gradOutput)
+   if self.gradInput then
+      return input.nn.SpatialFullConvolution_updateGradInput(self, input, gradOutput)
+   end
+end
+function SpatialFullConvolution:accGradParameters(input, gradOutput, scale)
+   return input.nn.SpatialFullConvolution_accGradParameters(self, input, gradOutput, scale)
+end
+

SpatialFullConvolutionMap.lua

@@ -0,0 +1,62 @@
+local SpatialFullConvolutionMap, parent = torch.class('nn.SpatialFullConvolutionMap', 'nn.Module')
+
+function SpatialFullConvolutionMap:__init(conMatrix, kW, kH, dW, dH)
+   parent.__init(self)
+
+   dW = dW or 1
+   dH = dH or 1
+
+   self.kW = kW
+   self.kH = kH
+   self.dW = dW
+   self.dH = dH
+   self.connTable = conMatrix
+   self.nInputPlane = self.connTable:select(2,1):max()
+   self.nOutputPlane = self.connTable:select(2,2):max()
+
+   self.weight = torch.Tensor(self.connTable:size(1), kH, kW)
+   self.gradWeight = torch.Tensor(self.connTable:size(1), kH, kW)
+
+   self.bias = torch.Tensor(self.nOutputPlane)
+   self.gradBias = torch.Tensor(self.nOutputPlane)
+   
+   self:reset()
+end
+
+function SpatialFullConvolutionMap:reset(stdv)
+   if stdv then
+      stdv = stdv * math.sqrt(3)
+      self.weight:apply(function()
+			   return torch.uniform(-stdv, stdv)
+			end)
+      self.bias:apply(function()
+			 return torch.uniform(-stdv, stdv)
+		      end)
+   else
+      local ninp = torch.Tensor(self.nOutputPlane):zero()
+      for i=1,self.connTable:size(1) do ninp[self.connTable[i][2]] =  ninp[self.connTable[i][2]]+1 end
+      for k=1,self.connTable:size(1) do
+         stdv = 1/math.sqrt(self.kW*self.kH*ninp[self.connTable[k][2]])
+         self.weight:select(1,k):apply(function() return torch.uniform(-stdv,stdv) end)
+      end
+      for k=1,self.bias:size(1) do
+         stdv = 1/math.sqrt(self.kW*self.kH*ninp[k])
+         self.bias[k] = torch.uniform(-stdv,stdv)
+      end
+
+   end
+end
+
+function SpatialFullConvolutionMap:updateOutput(input)
+   input.nn.SpatialFullConvolutionMap_updateOutput(self, input)
+   return self.output
+end
+
+function SpatialFullConvolutionMap:updateGradInput(input, gradOutput)
+   input.nn.SpatialFullConvolutionMap_updateGradInput(self, input, gradOutput)
+   return self.gradInput
+end
+
+function SpatialFullConvolutionMap:accGradParameters(input, gradOutput, scale)
+   return input.nn.SpatialFullConvolutionMap_accGradParameters(self, input, gradOutput, scale)
+end

TanhShrink.lua

@@ -0,0 +1,20 @@
+local TanhShrink, parent = torch.class('nn.TanhShrink','nn.Module')
+
+function TanhShrink:__init()
+   parent.__init(self)
+   self.tanh = nn.Tanh()
+end
+
+function TanhShrink:updateOutput(input)
+   local th = self.tanh:updateOutput(input)
+   self.output:resizeAs(input):copy(input)
+   self.output:add(-1,th)
+   return self.output
+end
+
+function TanhShrink:updateGradInput(input, gradOutput)
+   local dth = self.tanh:updateGradInput(input,gradOutput)
+   self.gradInput:resizeAs(input):copy(gradOutput)
+   self.gradInput:add(-1,dth)
+   return self.gradInput
+end

WeightedMSECriterion.lua

@@ -0,0 +1,31 @@
+local WeightedMSECriterion, parent = torch.class('nn.WeightedMSECriterion','nn.MSECriterion')
+
+function WeightedMSECriterion:__init(w)
+   parent.__init(self)
+   self.weight = w:clone()
+   self.buffer = torch.Tensor()
+end
+
+function WeightedMSECriterion:updateOutput(input,target)
+	self.buffer:resizeAs(input):copy(target)
+	if input:dim() - 1 == self.weight:dim() then
+		for i=1,input:size(1) do
+			self.buffer[i]:cmul(self.weight)
+		end
+	else
+		self.buffer:cmul(self.weight)
+	end
+	return input.nn.MSECriterion_updateOutput(self, input, self.buffer)
+end
+
+function WeightedMSECriterion:updateGradInput(input, target)
+   self.buffer:resizeAs(input):copy(target)
+	if input:dim() - 1 == self.weight:dim() then
+		for i=1,input:size(1) do
+			self.buffer[i]:cmul(self.weight)
+		end
+	else
+		self.buffer:cmul(self.weight)
+	end
+   return input.nn.MSECriterion_updateGradInput(self, input, self.buffer)
+end

generic/L1Cost.c

@@ -0,0 +1,49 @@
+#ifndef TH_GENERIC_FILE
+#define TH_GENERIC_FILE "generic/L1Cost.c"
+#else
+
+static int nn_(L1Cost_updateOutput)(lua_State *L)
+{
+  THTensor *input = luaT_checkudata(L, 2, torch_Tensor);  
+  accreal sum;
+
+  sum = 0;
+  TH_TENSOR_APPLY(real, input, sum += fabs(*input_data););
+
+  lua_pushnumber(L, sum);
+  lua_setfield(L, 1, "output");
+
+  lua_pushnumber(L, sum);
+  return 1;
+}
+
+static int nn_(L1Cost_updateGradInput)(lua_State *L)
+{
+  THTensor *input = luaT_checkudata(L, 2, torch_Tensor);
+  THTensor *gradInput = luaT_getfieldcheckudata(L, 1, "gradInput", torch_Tensor);
+
+  THTensor_(resizeAs)(gradInput, input);
+  TH_TENSOR_APPLY2(real, gradInput, real, input,
+                   if (*input_data > 0)
+                     *gradInput_data = 1;
+                   else if (*input_data < 0)
+                     *gradInput_data = -1;
+                   else
+                     *gradInput_data = 0;);
+  return 1;
+}
+
+static const struct luaL_Reg nn_(L1Cost__) [] = {
+  {"L1Cost_updateOutput", nn_(L1Cost_updateOutput)},
+  {"L1Cost_updateGradInput", nn_(L1Cost_updateGradInput)},
+  {NULL, NULL}
+};
+
+static void nn_(L1Cost_init)(lua_State *L)
+{
+  luaT_pushmetatable(L, torch_Tensor);
+  luaT_registeratname(L, nn_(L1Cost__), "nn");
+  lua_pop(L,1);
+}
+
+#endif

generic/SpatialConvolutionMap.c

@@ -18,7 +18,7 @@ static int nn_(SpatialConvolutionMap_updateOutput)(lua_State *L)
   THTensor *output = luaT_getfieldcheckudata(L, 1, "output", torch_Tensor);
 
   luaL_argcheck(L, input->nDimension == 3, 2, "3D tensor expected");
-  luaL_argcheck(L, input->size[0] == nInputPlane, 2, "invalid number of input planes");
+  luaL_argcheck(L, input->size[0] >= nInputPlane, 2, "invalid number of input planes");
   luaL_argcheck(L, input->size[2] >= kW && input->size[1] >= kH, 2, "input image smaller than kernel size");
 
   THTensor_(resize3d)(output, nOutputPlane,