updates to cfa, etc.

contractive_feature_analysis/cos_experiment_condor/feature_extractor.py

@@ -10,14 +10,17 @@ def __init__(self, W, b):
 
     def redo_theano(self):
         X = T.matrix()
-        H = T.tanh(T.dot(X,self.W)+self.b)
+        H = self(X)
         self.extract = function([X],H)
 
         #number examples x number hiddens x number visibles
         J = (1.-T.sqr(H)).dimshuffle(0,1,'x') * self.W.dimshuffle('x',1,0)
         self.jacobian_of_expand = function([X],J)
     #
 
+    def __call__(self, X):
+        return T.tanh(T.dot(X, self.W)+self.b)
+    #
 
     @classmethod
     def make_from_examples(cls, X, low, high, directed = True):

contractive_feature_analysis/cos_experiment_condor/stage_1_prepper_no_pca.py

@@ -9,11 +9,13 @@
 from pylearn2.datasets.cos_dataset import CosDataset
 from pylearn2.datasets.dense_design_matrix import DenseDesignMatrix
 from feature_extractor import TanhFeatureExtractor
+from theano import config
+floatX = config.floatX
 
 print 'making dataset'
 t1 = time.time()
 data = CosDataset()
-X = data.get_batch_design(50000)
+X = N.cast[floatX](data.get_batch_design(50000))
 data = DenseDesignMatrix(X)
 t2 = time.time()
 print (t2-t1),' seconds'
@@ -31,6 +33,7 @@
 pca_model = CovEigPCA(num_components = pca_dim)
 pca_model.train(X)
 pca_model.W.set_value(N.cast['float32'](N.identity(X.shape[1])))
+assert pca_model.get_weights().shape[1] == pca_dim
 pca_model.mean.set_value(N.cast['float32'](N.zeros(X.shape[1])))
 t2 = time.time()
 print (t2-t1),' seconds'
@@ -71,25 +74,52 @@
 serial.save(components+'/num_examples.pkl',num_examples)
 serial.save(components+'/expanded_dim.pkl',expanded_dim)
 
-if Z.shape[0] != Z.shape[1]:
-    print "fuck! Z.shape = "+str(Z.shape)
 
-"""print 'done, checking result'
+print 'done, checking result'
 
 #checks
 from theano import function
 import theano.tensor as T
 pca_input = T.matrix()
+assert pca_input.dtype == floatX
 
 del whitener
 whitener = serial.load(components+'/whitener.pkl')
 
 out = whitener(pca_input)
+assert out.dtype == floatX
 out_func = function([pca_input],out)
-g1 = out_func(N.cast['float32'](g1))
+test = out_func((g1))
 
-print g1[0:5,0:5]
+#print g1[0:5,0:5]
 
+
+
+"""g1 -= whitener.mean.get_value()
+print 'after manual mean subtract, mean is'
 mu = g1.mean(axis=0)
-print (mu.min(),mu.max())
+print (mu.min(), mu.max())
+g1 = N.dot(g1,whitener.get_weights())
+print 'after manual whitening, mean is '
 """
+
+mu = test.mean(axis=0)
+print (mu.min(), mu.max())
+
+print 'standard deviation'
+std = test.std(axis=0)
+print (std.min(), std.max())
+
+whitener.W.set_value(whitener.W.get_value() / std )
+
+test = out_func(g1)
+
+
+mu = test.mean(axis=0)
+print (mu.min(), mu.max())
+
+print 'standard deviation'
+std = test.std(axis=0)
+print (std.min(), std.max())
+
+serial.save(components+'/whitener.pkl',whitener)

contractive_feature_analysis/cos_experiment_condor/stage_4_finisher.py

@@ -1,4 +1,4 @@
-job_name = 'cfa_olshausen'
+job_name = 'cfa_cos_tanh'
 
 from pylearn2.utils import serial
 import SkyNet
@@ -13,14 +13,15 @@
 chunk_size = serial.load(components+'/chunk_size.pkl')
 batch_size = serial.load(components+'/batch_size.pkl')
 expanded_dim = serial.load(components+'/expanded_dim.pkl')
+whitened_dim = serial.load(components+'/whitener.pkl').get_weights().shape[1]
 
 instability_matrices = SkyNet.get_dir_path('instability_matrices')
 components = SkyNet.get_dir_path('components')
 
 assert num_examples % chunk_size == 0
 num_chunks = num_examples / chunk_size
 
-G = N.zeros((expanded_dim, expanded_dim) )
+G = N.zeros((whitened_dim, whitened_dim) )
 
 print 'Summing up instability matrices'
 for b in xrange(0,num_examples,chunk_size):

contractive_feature_analysis/mnist_experiment_condor/stage_1_prepper.py

@@ -85,3 +85,5 @@
 
 mu = g1.mean(axis=0)
 print (mu.min(),mu.max())
+std = g1.mean(axis=0)
+print (std.min(),std.max())

exploring_estimation_criteria/cos_reconsE_lnce.yaml

@@ -1,10 +1,19 @@
+#Was in the course of translating this to use SGD instead of DefaultTrainingAlgorithm. Realized it'll be kind of complicated
+#to restore since local_noise_ebm was accidentally deleted when I removed it from the public repo (I moved it into galatea,
+#deleted the old class repo, then accidentally overwrote the copy in
+#galatea with a git command that didn't do what I expected it to (fuck you, git), which at that time was the only copy)
+#
+# Also, it looks like this might not be the best script to convert, since it says "different_examples" : 1.
+#
 !obj:pylearn2.scripts.train.Train {
     "dataset": !obj:pylearn2.datasets.cos_dataset.CosDataset {},
-    "model": !obj:pylearn2.models.local_noise_ebm.LocalNoiseEBM {
-                "nvis" : 2,
-                "nhid" : 400,
-                "init_bias_hid" : 0.0,
-                "irange"  : 2.5,
+    "model": !obj:galatea.models.febm.FEBM {
+                "energy_function": !obj:galatea.energy_functions.scratch.recons_model_1 {
+                        "nvis" : 2,
+                        "nhid" : 400,
+                        "init_bias_hid" : 0.0,
+                        "irange"  : 2.5
+                        }
                 "init_noise_var"    : 2.0,
                 "min_misclass"  : .05,
                 "max_misclass"  : .5,
@@ -17,10 +26,11 @@
                 "init_vis_prec" : 1.,
                 "learn_vis_prec" : 1.,
                 "vis_prec_lr_scale" : .001,
-                "energy_function" : "mse autoencoder",
+
                 "init_delta" : -0.5
         },
     "algorithm": !obj:pylearn2.training_algorithms.default.DefaultTrainingAlgorithm {
+               "learning_rate" : .0002,
                "batch_size" : 5,
                "batches_per_iter" : 1000,
                "monitoring_batches" : 10,

models/febm.py

@@ -1,8 +1,34 @@
-
+import theano.tensor as T
+from theano import function
 
 class FEBM:
     """Free Energy-Based Model: An Energy-Based Model with no latent variables """
 
     def __init__(self, energy_function):
-        self.energy_function
+        self.energy_function = energy_function
+        self.batches_seen = 0
+        self.examples_seen = 0
+    #
+
+    def free_energy(self, X):
+        return self.energy_function(X)
+    #
+
+    def score(self, X):
+        return self.energy_function.score(X)
+    #
+
+    def censor_updates(self, updates):
+        self.energy_function.censor_updates(updates)
+    #
+
+    def get_params(self):
+        return self.energy_function.get_params()
+    #
+
+    def redo_theano(self):
+        X = T.matrix()
 
+        self.E_X_batch_func = function([X],self.energy_function(X))
+    #
+#