various stuff

energy_functions/energy_function.py

@@ -11,7 +11,7 @@ def score(self, X):
 
         X_name = 'X' if X.name is None else X.name
 
-        E = self(X)
+        E = self.free_energy(X)
 
         #There should be one energy value for each example in the batch
         assert len(E.type.broadcastable) == 1

energy_functions/scratch.py

@@ -59,6 +59,14 @@ def redo_everything(self):
         #
     #
 
+    def get_weights_format(self):
+        return ['v','h']
+    #
+
+    def get_weights(self, borrow = False):
+        return self.W.get_value(borrow = borrow)
+    #
+
     def encode(self, X):
         X_name = 'X' if X.name is None else X.name
         H = T.nnet.sigmoid(T.dot(X, self.W) + self.bias_hid)

exploring_estimation_criteria/cos_plot.py

@@ -1,29 +1,33 @@
 import numpy as N
 from theano import config
 floatX = config.floatX
+from pylearn2.datasets.cos_dataset import CosDataset
+from pylearn2.gui.graph_2D import Graph2D, HeatMap
+import theano.tensor as T
+from theano import function
+from theano.printing import Print
 
 rows = 500
 cols  = 1000
 
 xmin = -6.2
 xmax = 6.2
 
-ymin = -3.3
-ymax = 3.3
-
-from pylearn2.config import yaml_parse
 import sys
 from pylearn2.utils import serial
 
 model = serial.load(sys.argv[1])
 model.redo_theano()
-dataset = yaml_parse.load(model.dataset_yaml_src)
+dataset = CosDataset()
 
+print 'examples seen: '+str(model.examples_seen)
 
 PDF = 0    #pdf
 GPDF = 1   #gradient of pdf
 GDPDF = 2  #gradient direction of pdf
 ENERGY = 3 #energy
+SCORE = 4  #score
+SCORED = 5 #score direction
 
 if len(sys.argv) == 2 or sys.argv[2] == '--pdf':
     t = PDF
@@ -33,127 +37,113 @@
     t = GDPDF
 elif sys.argv[2] == '--energy':
     t = ENERGY
+elif sys.argv[2] == '--score':
+    t = SCORE
+elif sys.argv[2] == '--scored':
+    t = SCORED
+else:
+    raise ValueError('unknown flag '+sys.argv[2])
 
+g = Graph2D( shape = (rows, cols), xlim=(xmin,xmax), ycenter = 0. )
 
 
-def make_img(f):
-    rval = N.zeros((rows,cols,1))
-
-    for i in xrange(cols):
-        print 'col '+str(i)
-        x = xmin + (xmax-xmin)*float(i)/float(cols-1)
-        ys = [ymin+(ymax-ymin)*float(j)/float(rows-1) for j in xrange(rows) ]
-        assert len(ys) == rows
-
-        xs = N.zeros((rows,1))+x
-        assert xs.shape[0] == rows
-        assert xs.shape[1] == 1
-
-        ys = N.asarray(ys)
-        ys = N.asarray([ys]).T
-        assert ys.shape[0] == rows
-        assert ys.shape[1] == 1
-
-        mat = N.hstack( (xs,ys) )
-
-        mat = N.cast[floatX](mat)
-
-        assert mat.shape[0] == rows
-        assert mat.shape[1] == 2
-
-        mf = f(mat)
-
-        assert mf.shape[0] == 500
-        assert len(mf.shape) == 1
+def energy_normalizer(I):
+    I -= I.min()
+    I /= I.max()
+    return  (I*2.0)-1.0
 
-        #try:
-        rval[:,i,0] = mf
-        #except ValueError, e:
-        #    print rval[:,i,0].shape
-        #    print f(mat).shape
-        #    raise e
+def pdf_normalizer(I):
+    assert I.min() >= 0.0
+    I /= I.max()
 
-    assert not N.any(N.isinf(rval))
-    assert not N.any(N.isnan(rval))
+    for i in xrange(I.shape[0]):
+        for j in xrange(I.shape[1]):
+            if N.any(N.isnan(I[i,j,:]) + N.isinf(I[i,j,:])):
+                I[i,j,0] = 1.0
+                I[i,j,1] = 0.0
+                I[i,j,2] = 0.0
 
-    #rval -= rval.mean()
-    #return rval
+    return (I*2.0)-1.0
 
-    if t == ENERGY:
-        rval -= rval.min()
-        rval /= rval.max()
-        return rval
+def gpdf_normalizer(I):
+    I /= N.abs(I).max()
+    return I
 
-    assert rval.min() >= 0.0
-    mx = rval.max()
+def gdpdf_normalizer(I):
+    return I
 
-    rval /= mx
-    rval *= 2.0
-    rval -= 1.0
+X = T.matrix()
 
+class FuckYouTheano:
+    def __init__(self, join_should_be_able_to_do_this):
+        self.f = join_should_be_able_to_do_this
 
-    if t == PDF:
+    def __call__(self, X):
+        rval = N.zeros((X.shape[0],3),dtype=floatX)
+        rval[:,0:2] = self.f(X)
         return rval
 
-    rval2 = N.zeros((rval.shape[0]-1,rval.shape[1]-1,3))
-
-    for i in xrange(rval2.shape[0]):
-        for j in xrange(rval2.shape[1]):
-            #rval2[i,j,0] = N.sign(rval[i+1,j,0]-rval[i,j,0])
-            #rval2[i,j,1] = N.sign(rval[i,j+1,0]-rval[i,j,0])
-
-            rval2[i,j,0] = rval[i+1,j,0]-rval[i,j,0]
-            rval2[i,j,1] = rval[i,j+1,0]-rval[i,j,0]
-            if t == GDPDF:
-                rval2[i,j,:] /= (1e-15+N.sqrt((rval2[i,j,:] ** 2.).sum()))
-            #
-        #
-    #
-
-    if t == GPDF:
-        rval2 /= N.abs(rval2).max()
-
+def grad_func( pdf ):
+    grad = T.grad(pdf.sum(), X)
+    return FuckYouTheano(function([X],grad))
 
-    return rval2
+def grad_dir_func( pdf ):
+    grad = T.grad(pdf.sum(), X)
+    grad = Print('before',attrs=['min','max'])(grad)
+    grad /= T.sqrt(1e-15+T.sum(T.sqr(grad),axis=1).dimshuffle(0,'x'))
+    grad = Print('after',attrs=['min','max'])(grad)
+    return FuckYouTheano(function([X],grad))
 
-def pdf(mat):
-    rval = model.E_X_batch_func(mat)
-    #assert not N.any(N.isinf(rval))
-    #assert not N.any(N.isnan(rval))
-    assert rval.shape[0] == mat.shape[0]
-    assert len(rval.shape) == 1
 
-    #print (rval.min(),rval.max())
+if t not in [ENERGY, SCORE, SCORED]:
+    g.components.append(HeatMap( f = function([X], model.free_energy(X)),  normalizer = None ))
+    offset = g.render().mean()
+#
 
-    if t != ENERGY:
-        rval = N.exp(-rval)
-    #assert not N.any(N.isinf(rval))
-    #assert not N.any(N.isnan(rval))
-
-    return rval
+if t == ENERGY:
+    df = dataset.free_energy_func
+    mfe = model.free_energy(X)
+    mf = function([X],mfe)
+    normalizer = energy_normalizer
+elif t == PDF:
+    df = dataset.pdf_func
+
+    mfe = model.free_energy(X)
+    mfe = Print('model free energy',attrs=['min','max'])(mfe)
+    mf = function([X], T.exp(-mfe+offset))
+
+    normalizer = pdf_normalizer
+elif t == GPDF:
+    df = grad_func(dataset.pdf(X))
+    mf = grad_func(T.exp(-model.free_energy(X)+offset))
+
+    normalizer = gpdf_normalizer
+elif t == GDPDF:
+    df = grad_dir_func(dataset.pdf(X))
+    mf = grad_dir_func(T.exp(-model.free_energy(X)+offset))
+
+    normalizer = gdpdf_normalizer
+elif t == SCORE:
+    df = grad_func(- dataset.free_energy(X))
+    mf = grad_func( - model.free_energy(X))
+
+    normalizer = gpdf_normalizer
+elif t == SCORED:
+    df = grad_dir_func(- dataset.free_energy(X))
+    mf = grad_dir_func( - model.free_energy(X))
+
+    normalizer = gdpdf_normalizer
+else:
+    assert False
 
 
+g.components.append(HeatMap(f = df, normalizer = normalizer))
+dimg = g.render()
 
-print 'making dataset image'
-if t == ENERGY:
-    dimg = make_img(dataset.energy)
-else:
-    dimg = make_img(dataset.pdf)
-print 'making model image'
-mimg = make_img(pdf)
-
-mn = min([dimg.min(),mimg.min()])
-dimg -= mn
-mimg -= mn
-mx = max([dimg.max(),mimg.max()])
-dimg /= mx
-mimg /= mx
-
-dimg *= 2.
-dimg -= 1.
-mimg *= 2.
-mimg -= 1.
+g.components.pop()
+g.components.append(HeatMap(f = mf, normalizer = normalizer))
 
+mimg = g.render()
 
 from pylearn2.gui.patch_viewer import PatchViewer
 

exploring_estimation_criteria/cos_reconsE_nce.yaml

@@ -4,6 +4,7 @@
     "model": !obj:pylearn2.models.normalized_ebm.NormalizedEBM {
                 "init_logZ" : 0.,
                 "learn_logZ" : 1,
+                "logZ_lr_scale" : .001,
                 "ebm": !obj:galatea.models.febm.FEBM {
                 "energy_function": !obj:galatea.energy_functions.scratch.recons_model_1 {
                         "nvis" : 2,

exploring_estimation_criteria/cos_reconsE_sm.yaml

@@ -16,7 +16,7 @@
                         }
         },
     "algorithm": !obj:pylearn2.training_algorithms.sgd.SGD {
-               "learning_rate" : .02,
+               "learning_rate" : .002,
                "batch_size" : 5,
                "batches_per_iter" : 1000,
                "monitoring_batches" : 10,

exploring_estimation_criteria/cos_reconsE_smd.yaml

@@ -5,29 +5,29 @@
                 "energy_function": !obj:galatea.energy_functions.scratch.recons_model_1 {
                         "nvis" : 2,
                         "nhid" : 400,
-                        "irange"  : .5,
-                        "init_bias_hid" : 0.0,
+                        "irange"  : 1.8,
+                        "init_bias_hid" : 0.,
 
-                        "init_vis_prec" : 5.,
-                        "vis_prec_lr_scale" : .1,
+                        "init_vis_prec" : 1.,
+                        "vis_prec_lr_scale" : .001,
                         "learn_vis_prec" : 1.,
 
-                        "init_delta" : -0.5
+                        "init_delta" : 0.0
                         }
         },
     "algorithm": !obj:pylearn2.training_algorithms.sgd.SGD {
                "learning_rate" : .0002,
-               "batch_size" : 5,
-               "batches_per_iter" : 1000,
-               "monitoring_batches" : 10,
+               "batch_size" : 50,
+               "batches_per_iter" : 10000,
+               "monitoring_batches" : 100,
                "monitoring_dataset" : !obj:pylearn2.datasets.cos_dataset.CosDataset {},
                "cost" : !obj:pylearn2.costs.ebm_estimation.SMD {
                         "corruptor": !obj:pylearn2.corruption.GaussianCorruptor {
-                                "stdev": .01
+                                "stdev": .1
                         }
                }
         },
-    "save_path": "cos_recons_E_smd.pkl"
+    "save_path": "cos_reconsE_smd.pkl"
 }
 
 

exploring_estimation_criteria/experiment_a.yaml

@@ -1,5 +1,4 @@
-#this learns a model with the right gradient direction everywhere, but totally wrong global structure
-
+#like snapshot 1, but with few hidden units, to see what happens less capacity
 !obj:pylearn2.scripts.train.Train {
     "dataset": !obj:pylearn2.datasets.cos_dataset.CosDataset {},
     "model": !obj:galatea.models.local_noise_ebm.LocalNoiseEBM {

models/febm.py

@@ -26,6 +26,14 @@ def get_params(self):
         return self.energy_function.get_params()
     #
 
+    def get_weights(self):
+        return self.energy_function.get_weights()
+    #
+
+    def get_weights_format(self):
+        return self.energy_function.get_weights_format()
+    #
+
     def redo_theano(self):
         X = T.matrix()
 

models/local_noise_ebm.py

@@ -171,6 +171,9 @@ def p_h_given_v(self, V):
 
         return T.nnet.sigmoid(self.b + T.dot(V,self.W))
 
+    def free_energy(self, V):
+        return self.batch_free_energy(V)
+
     def batch_free_energy(self, V):
 
         if self.energy_function == 'gaussian-binary rbm':
@@ -279,7 +282,7 @@ def redo_theano(self):
         #corrupted = corrupted * norm_ratio_shuffled
         #corrupted.name = 'postnorm_corrupted'
 
-
+        print "NOT USING NORM RESCALING"
 
         self.corruption_func = function([X],corrupted)
 

recons_srbm/make_dataset_2M.py

@@ -1,6 +1,6 @@
-from framework.utils import serial
-from framework.datasets import cifar10
-from framework.datasets import preprocessing
+from pylearn2.utils import serial
+from pylearn2.datasets import cifar10
+from pylearn2.datasets import preprocessing
 
 train = cifar10.CIFAR10(which_set="train")