GBRBM_main.m

% 
%     Gaussian-Bernoulli Restricted Boltzmann Machine Using
%           Minimum Probability Flow Learning
%     Main Procedure
%     Initializes the GBRMB, loads data to learn, calls the appropriate
%           learning functions.
%
%     Steven Munn
%     email: stevenjlm@berkeley.edu

% Commented out, used for running optimization
%function v_afterCD2

addpath ../3rd_party_code/minFunc % Minimizer function
addpath grbm % General function
addpath learning

clear; % CLEARS ALL PREVIOUS DATA!
clc; % CLEARS ALL CONSOLE DATA!

%% Parameters,
% General structure
grbm.nPixels = 8^2; % number of units/pixels (needs to have a integer sqrt)
grbm.ImageDimensions=sqrt(grbm.nPixels); % dimesions of the training pictures ImageSz by ImageSz
grbm.N = 2^13; % number of training samples
grbm.nV=3*grbm.nPixels; % Number of visible units in the RBM
grbm.nH=324; % Number of hidden units in the RBM

% Learning Parameters
grbm.UseSingleSigma=1; % Use only 1 sigma for learning
grbm.BatchSize=2^13; % Batch size=Number of images to use at any given time
grbm.ZeroMean=1;
grbm.MakeStdOne=1;
grbm.Verbose=1;
grbm.SegmentSz=2^6;

% Sampling parameters
grbm.BurnIn=1000;
grbm.SpaceTweenSamples=1E1;
grbm.SingleChain=0; % Set to zero if using PT
grbm.ParallelTempering=1; % Set to 1 to use PT
grbm.nPTChains=5;
grbm.SwapInterval=20;
grbm.debugSampler=0;

% Minimization Routine options [See Readme]
minf_options = [];
minf_options.maxlinesearch = 50;
minf_options.TolX=1E-9;
minf_options.TolFun=1E-1;

%% Load and Preprosses Training Data
overall=tic();
load('C:\Users\stevenjlm\Google Drive\biophysics\Myproj\CD_grbm\dataparse\8by8data_W1neg3.mat');
Dall_VbN=Dall(1:3*grbm.nPixels,1:grbm.N);
clear Dall;
% Data generated by get_data_cifar.m in the Myproj\CD_grbm\dataparse\ dir

% Check you have the correct data
if size(Dall_VbN,2)~=grbm.N
    error('The data you are loading for training is probably not what you expected');
end

% number of minibatches
grbm.NumMiniBatch=floor(grbm.N / grbm.BatchSize);
% truncate Dall
Dall_VbN=Dall_VbN(:, 1 : grbm.NumMiniBatch * grbm.BatchSize);
% Get the number of samples left
grbm.N=size(Dall_VbN,2);

% Preprosses
if grbm.ZeroMean
    Dall_VbN=bsxfun(@minus,Dall_VbN,mean(Dall_VbN));
end
if grbm.MakeStdOne
    Dall_VbN=bsxfun(@rdivide,Dall_VbN,std(Dall_VbN')'); %#ok<UDIM>
end
if grbm.ParallelTempering
    grbm.ithMean_Vb1=mean(Dall_VbN,2);
    grbm.ithSigma_Vb1=std(Dall_VbN')';
end

%% Initialize GBRBM

Weights_HbV=0.001*randn(grbm.nH, grbm.nV);
HBias_Hb1=zeros( grbm.nH, 1);
VBias_Vb1=zeros( grbm.nV, 1);
Sigmas_Vb1=ones( grbm.nV, 1);

% Samples for initial batch
%samples=sample_grbm_v2(Wl, hbiasl, vbiasl, sigmasl, bsize, 100);
grbm.steps=1;
[Samples_VbBS,AllChains_VbNP]=sample_grbm( Weights_HbV,VBias_Vb1, HBias_Hb1, Sigmas_Vb1, grbm,...
    Dall_VbN(:,(end-grbm.nPTChains+1):end));

Theta=[Weights_HbV(:);...
    HBias_Hb1(:);...
    VBias_Vb1(:);...
    Sigmas_Vb1(:)];
%% Train the GRBM
grbm.start=1;
grbm.epochs=20;

minf_options.MaxFunEvals=10;

for i=grbm.start:grbm.epochs
    for j=1:grbm.NumMiniBatch
        
        if grbm.Verbose
            fprintf( 'Learning step %d of %d\n', j,grbm.NumMiniBatch );
        end
        
        % Set the number of Minfunc evaulations
%         minf_options.MaxFunEvals=BuildUp(i);
        
        % Run Minfunc
        ThetaNew = minFunc( @k_dk_grbm, Theta(:), minf_options,...
            Samples_VbBS,...
            Dall_VbN(:,1+grbm.BatchSize*(j -1):1+grbm.BatchSize*j-1),...
            Theta,...
            grbm);
        
        Theta=ThetaNew;
        % Parse
        [Weights_HbV,...
        VBias_Vb1,...
        HBias_Hb1,...
        Sigmas_Vb1]=theta_parser( ThetaNew, grbm);
        
        grbm.BurnIn=0;
        grbm.SpaceTweenSamples=0;
        % Get new Data Samples from the Model
        [Samples_VbBS,AllChains_VbNP]=sample_grbm( Weights_HbV,VBias_Vb1, HBias_Hb1, Sigmas_Vb1, grbm,AllChains_VbNP);
        %[Samples_VbBS,AllChains_VbNP]=sample_grbm( Weights_HbV,VBias_Vb1, HBias_Hb1, Sigmas_Vb1, grbm);
 
        % Calculate the Reconstruction error
        Error=recon_error( Weights_HbV, HBias_Hb1, VBias_Vb1, Sigmas_Vb1,...
            Dall_VbN(:,1+grbm.BatchSize*(j-1):1+grbm.BatchSize*j-1));

        fprintf( 2,'\nReconstruction error, %f And relative err, %f\n', Error, Error/grbm.BatchSize);
    end
    fprintf('Completed epoch %d\n',i);
    save(['results\0812\Weights_',num2str(i),'.mat'], 'Weights_HbV');
    save(['results\0812\Samples_',num2str(i),'.mat'], 'Samples_VbBS');
%     minf_options.MaxFunEvals=input('Max Function eval?');
end
%end
overall=toc(overall);
disp(overall);