BatchVerMo

.gitignore

@@ -1,3 +1,5 @@
+**.png
+
 *.mat
 cvx/
 *.avi

BatchVerMO/A2C.m

@@ -0,0 +1,110 @@
+function ACS_temp=A2C(Ysignal, A, options)
+% ACS=A2C2A(ACS, File, A, options)
+% General Description:
+%      This function is designed to extract C in background-subtracted signal
+%      from A(not Amask) in Ysignal.
+%      The method relies on CNMF-E's conference script.
+%      For each successful ai extraction, peel A*C off Ysignal.
+%      Iterate for all the neurons indicated by A.
+% Input: 
+% Ysignal(background-subtracted), with dimenstion of [d1*d2,T];
+% A is also used for center calculation(for subtraction previous signal).
+% options: struct data of paramters/options
+    %    d1:     number of rows
+    %    d2:     number of columns
+    %    gSiz:   maximum size of a neuron
+    %    nb:     left-over from the previous version, number of knots in
+    %            modeling background,=1.
+    %    min_corr: minimum threshold of correlation for segementing neurons
+    %    deconv_options
+    %    deconv_flag    
+% Output:
+% ACS_temp is a structure with three fields: A,C,and C's
+%       standard deviation(STD). This structure is used for merging in later steps in cnmf_e-BatchVer. 
+
+% Modified from "greedyROI_endoscope"
+% Main dependences are "extract_a", "extract_c" in CNMF-E BatchVer, and "com" for neuron center by Pengcheng Zhou.
+
+% Shijie Gu, [email protected]
+
+%% parameters and preparations
+
+d1 = options.d1;        % image height
+d2 = options.d2;        % image width
+gSiz = options.gSiz;    % average size of neurons
+
+Acenter = round(com(A, d1, d2)); %nr x 2 matrix, with the center of mass coordinates
+
+Ysignal(isnan(Ysignal)) = 0;     % remove nan values
+Ysignal = double(Ysignal);
+T = size(Ysignal, 2);
+
+deconv_options_0= options.deconv_options;
+deconv_flag = options.deconv_flag;
+
+K = size(A,2);
+Cin = zeros(K, T);      % temporal components
+Cin_raw = zeros(K, T);  % temporal components
+STD = zeros(1,K);       % standard deviation
+
+%% start initialization
+for k = 1:K
+    r=Acenter(k,1);
+    c=Acenter(k,2);
+
+    % Use center point data to roughly check whether this is a good starting point
+%    ind_p=sub2ind([d1 d2], r, c);
+%     y0 = Ysignal(ind_p, :);    
+%     y0_std = std(diff(y0));
+%     if max(diff(y0))< y0_std % signal is weak
+%         Ain(:,k)=A(:,k);       % If it is a "poor" ci, no problem, low STD will let not it contribute much to finalA.
+%         Cin(k,:)=y0;           % Since poor ci will get poor A that has bad shapes. Use normal A to fill in the place.
+%         STD(k)=std(y0);
+%         continue;
+%     end
+
+    % extract ci    
+    [ci_raw,ind_success_ci] = extract_c(Ysignal,[],A(:,k));
+    Cin_raw(k,:)=ci_raw;
+
+    if ~ind_success_ci          % If it is a "poor" ci, no need to subtract it before next neuron.
+        Cin(k,:)=ci_raw;
+        STD(k)=std(ci_raw);
+        continue;
+    end
+
+    if and(ind_success_ci,deconv_flag)
+        try
+            % deconv the temporal trace
+            [ci, ~, ~] = deconvolveCa(ci_raw, deconv_options_0);
+            % save this initialization
+            Cin(k, :) = ci;            
+            STD(k)=std(ci);
+            ci=ci';
+        catch
+            Cin(k, :)=ci_raw;
+            STD(k)=std(ci_raw);
+            ci=ci_raw;
+        end      
+    end
+
+    % select its neighbours to subtract some data, the box size is
+    %[2*gSiz+1, 2*gSiz+1]
+    % if the c is poor
+    ci_std = std(diff(ci));
+    if max(diff(ci))< ci_std % signal is weak
+        continue
+    end
+    rsub = max(1, -gSiz+r):min(d1, gSiz+r);
+    csub = max(1, -gSiz+c):min(d2, gSiz+c);
+    [cind, rind] = meshgrid(csub, rsub);
+    ind_nhood = sub2ind([d1, d2], rind(:), cind(:));
+    HY_box = Ysignal(ind_nhood, :);      
+    Ysignal(ind_nhood, :) = HY_box - A(ind_nhood,k)*ci_raw;  % update data
+end
+ACS_temp=struct('Cin',[],'Cin_raw',[],'STD',[]);
+ACS_temp.Cin = Cin;
+ACS_temp.Cin_raw=Cin_raw;
+ACS_temp.STD = STD;
+
+end

BatchVerMO/A2image.m

@@ -0,0 +1,62 @@
+function image=A2image(A,d1,d2,textOrNot,color,PoporNOt,handle)
+
+if nargin<4
+    textOrNot=false;
+end
+
+if nargin<5
+    color=[];
+end
+
+if nargin<6
+    PoporNOt=false;
+end
+
+A=A(:,any(A,1));
+k = size(A,2);
+sA = sum(A,1);
+A = bsxfun(@rdivide, A, sA)*prctile(sA, 5);
+C=ones(k,1);
+
+if isempty(color)==false
+    if or(strcmp(color,'magenta'),strcmp(color,'red'))
+        color_palet = 1-[1 0 1];
+    elseif strcmp(color,'green')
+        color_palet = 1-[0 1 0];
+    end
+        nColors = repmat(color_palet,size(A,2),1); 
+        Ir = diag(nColors(:,1));
+        Ig = diag(nColors(:,2));
+        Ib = diag(nColors(:,3));
+
+    Brainbow = 1-cat(3, A*Ir*C, A*Ig*C, A*Ib*C);
+    Brainbow = reshape(Brainbow,d1,d2,3);
+else
+    Brainbow = 1-A*C;
+    Brainbow(Brainbow<0)=0;
+    Brainbow = reshape(Brainbow,d1,d2);
+    %image=imshow(Brainbow); 
+end
+image=Brainbow;
+if PoporNOt==true
+    if nargin<7
+        figure
+    else
+        axes(handle)
+    end
+    imshow(Brainbow)
+end
+
+if textOrNot==true
+    Atemp=reshape(A,d1,d2,k);
+    Position=zeros(2,k);
+    for i=1:k
+        Atemp_=Atemp(:,:,i);
+        [row_ind,col_ind] = find(Atemp_>0);
+        Position(2,i)=mean(row_ind);
+        Position(1,i)=mean(col_ind);
+    end
+    text(Position(1,:),Position(2,:),cellstr(num2str((1:k)'))','Color','black')
+    F = getframe(gca);
+    image=F.cdata; 
+end

BatchVerMO/AllTraces.m

@@ -0,0 +1,27 @@
+function [AllC, boundary]=AllTraces(neuron_batch,type)
+if nargin<2||isempty(type);
+    type='rawsignal';
+end
+
+AllC=[];
+boundary=[];
+if strcmp(type,'rawsignal')
+    for i=1:length(neuron_batch)
+        AllC=[AllC neuron_batch(i).rawsignal];
+        boundary=[boundary size(neuron_batch(i).rawsignal,2)];
+    end
+elseif strcmp(type,'signal')
+    for i=1:length(neuron_batch)
+        AllC=[AllC neuron_batch(i).signal];
+        boundary=[boundary size(neuron_batch(i).signal,2)];
+    end
+end
+boundary=cumsum(boundary);
+end
+% %%
+% framesize=[];
+% framesizeC=[];
+% for i=1:length(neuron_batch)
+%     framesize=[framesize size(neuron_batch(i).signal,2)];
+%     framesizeC=[framesizeC size(neuron_batch(i).neuron.C,2)];   
+% end

BatchVerMO/AllTracesfromACS.m

@@ -0,0 +1,19 @@
+function [AllC, boundary]=AllTracesfromACS(ACS,rawOrNot)
+if nargin<2
+    rawOrNot=false;
+end
+AllC=[];
+boundary=[];
+for i=1:length(ACS)
+    if rawOrNot
+        AllC=[AllC ACS(i).Cin_raw];
+        boundary=[boundary size(ACS(i).Cin_raw,2)];
+    else
+        AllC=[AllC ACS(i).Cin];
+        boundary=[boundary size(ACS(i).Cin,2)];
+    end
+    boundary=cumsum(boundary);
+end
+
+
+end

BatchVerMO/BackgroundSub.m

@@ -0,0 +1,18 @@
+rr = ceil(neuron.options.gSiz * bg_neuron_ratio); 
+active_px = []; %(sum(IND, 2)>0);  %If some missing neurons are not covered by active_px, use [] to replace IND
+[Ybg, Ybg_weights] = neuron.localBG(Y, spatial_ds_factor, rr, active_px, neuron.P.sn, thresh); % estiamte local background.
+% subtract the background from the raw data.
+Ysignal = Y - Ybg;
+
+%% estimate noise 
+if ~isfield(neuron.P, 'sn') || isempty(neuron.P.sn)
+    %% estimate the noise for all pixels
+    b0 =zeros(size(Ysignal,1), 1);
+    sn = b0;
+    parfor m=1:size(neuron.A,1)
+        [b0(m), sn(m)] = estimate_baseline_noise(Ysignal(m, :));
+    end     
+    Ysignal = bsxfun(@minus, Ysignal, b0);
+    neuron.P.sn = sn;
+    %Ysignal = bsxfun(@minus, Ysignal, sn);
+end

BatchVerMO/BatchVer.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+#SBATCH -n 1
+#SBATCH --cpus-per-task=8
+#SBATCH --mem=20000
+#SBATCH -t 0-2:00
+#SBATCH --time-min=0-01:00
+#SBATCH -o /net/feevault/data0/shared/EmilyShijieShared/BatchResultTest/job_%A.out
+#SBATCH -e /net/feevault/data0/shared/EmilyShijieShared/BatchResultTest/job_%A.err
+
+cd /net/feevault/data0/shared/EmilyShijieShared/BatchResultTest/
+module add mit/matlab/2016b
+matlab -nodisplay -singleCompThread -r "addpath(genpath('/home/shijiegu/cnmf_e/'));\
+BatchVerScript"

BatchVerMO/BatchVerSLURM.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+#SBATCH -n 1
+#SBATCH --cpus-per-task=8
+#SBATCH --mem=150000
+#SBATCH -t 0-10:00
+#SBATCH --time-min=0-01:00
+#SBATCH -o /net/feevault/data0/shared/EmilyShijieShared/BatchResult/6922FirstFewDaysForBatchNewMerging/job_%A.out
+#SBATCH -e /net/feevault/data0/shared/EmilyShijieShared/BatchResult/6922FirstFewDaysForBatchNewMerging/job_%A.err
+cd /net/feevault/data0/shared/EmilyShijieShared/BatchResult/6922FirstFewDaysForBatchNewMerging/
+module add mit/matlab/2016b
+matlab -nodisplay -singleCompThread -r "addpath(genpath('/home/shijiegu/cnmf_e/'));\
+load(fullfile('/net/feevault/data0/shared/EmilyShijieShared/BatchResult/6922FirstFewDaysForBatchNewMerging/','PartOneOFcnmfeBatchVer.mat')); BatchVerTestVer "

BatchVerMO/BatchVerScript.m

@@ -0,0 +1,3 @@
+load(fullfile('/net/feevault/data0/shared/EmilyShijieShared/BatchResult/7030FirstFewDaysForBatch/','LogisticscnmfeBatchVer.mat'))
+addpath(genpath(codeDir));
+cnmfeBatchVer_ClusterPart

BatchVerMO/BatchVerTestVer.m

@@ -0,0 +1,77 @@
+%% 2 Merge similar neurons
+%%% Merge similar neurons based on spatial AND temporal correlation
+C_all=cat(2,ACS.Cin);
+
+Amask_temp=cat(2,A0s{1:2})>0;
+C_temp=C_all(1:size(Amask_temp,2),:);
+[Amask_temp,C_temp,ACS] = mergeAC(Amask_temp,C_temp,ACS,merge_thr);
+
+merge2start=1+size(A0s{1},2)+size(A0s{2},2);
+for i=3:length(samplelist_reduced)
+    Amask_temp=cat(2,Amask_temp,A0s{i})>0;
+
+    C_temp=[C_temp;C_all(merge2start:merge2start+size(A0s{i},2)-1,:)];
+
+    [Amask_temp,C_temp,ACS] = mergeAC(Amask_temp,C_temp,ACS,merge_thr);
+    merge2start=merge2start+size(A0s{i},2);
+end
+
+save([outputdir 'commonAcnmfeBatchVer.mat'],'-v7.3')
+%% 3 Determine uniqueA's
+As=cell(1,length(samplelist));
+STDs=cell(1,length(samplelist));
+parfor i=1:length(samplelist)
+    As{i}=ACS(i).Ain;
+    STDs{i}=ACS(i).STD;    
+end
+Afinal=ReducingA(As,STDs);
+save([outputdir 'uniqueAcnmfeBatchVer.mat'],'-v7.3')
+%% 5 Determine Afinal that will be used to extract C's in each file.
+
+%%% Some processes making Afinal nicer, modified from Pengcheng Zhou's
+%%% idea.
+for i=1:size(Afinal,2)
+    ai=Afinal(:,i);
+    temp = full(ai>quantile(ai, 0.5, 1));
+    ai(~temp(:)) = 0;
+    Afinal(:,i)=ai;
+end
+
+% Just in case some all zero A's got passed to this stage.
+nz_ind=any(Afinal);
+Afinal=Afinal(:,nz_ind);
+save([outputdir 'AfinalcnmfeBatchVer.mat'],'-v7.3')
+%% 6 "massive" procedure: Extract A from each file
+neuron_batch(length(filelist)) = struct('ind_del',[],'signal',[],'FileOrigin',[],'neuron',[]);
+
+parfor i= 1:length(filelist)  
+    mode='massive';
+    nam=fullfile(datadir,filelist(i).name);    
+    [~,neuron_batch(i)]=demo_endoscope2(gSig,gSiz,min_corr,min_pnr,FS,SSub,TSub,bg_neuron_ratio,nam,mode,[],Afinal,neuron_batch(i),convolveType);
+    neuron_batch(i).FileOrigin=filelist(i); % save origin(filelist)
+end
+fprintf('Massive extraction done.');
+save([outputdir 'MassivecnmfeBatchVer.mat'],'-v7.3')
+
+%neuron(length(filelist)) = struct('signal',[],'filelist',[]);
+%%% Partition between those neurons found in each file and those not.
+ind_del_final_cat=cat(2,neuron_batch.ind_del);
+ind_del_final=any(ind_del_final_cat,2);
+parfor i= 1:length(filelist)
+    neuron_batch(i).neuron.A=[neuron_batch(i).neuron.A(:,~ind_del_final) neuron_batch(i).neuron.A(:,ind_del_final)];
+    fprintf('A extraction done\n');
+    neuron_batch(i).neuron.C=[neuron_batch(i).neuron.C(~ind_del_final,:);neuron_batch(i).neuron.C(ind_del_final,:)];
+    neuron_batch(i).neuron.C_raw=[neuron_batch(i).neuron.C_raw(~ind_del_final,:);neuron_batch(i).neuron.C_raw(ind_del_final,:)];
+    fprintf('C extraction done\n');
+            %%% save each data i's signal into neuron(i).signal and
+    for j=1:size(neuron_batch(i).neuron.A,2)
+        jA=neuron_batch(i).neuron.A(:,j);
+        jC=neuron_batch(i).neuron.C(j,:);
+        neuron_batch(i).signal(j,:)=median(jA(jA>0)*jC);
+    end        
+    fprintf('neuron_batch %.0f extraction done\n', i);
+end
+fprintf('First %.0f neurons are successfully deconvolved in each files while those after that are missing in some files\n', sum(~ind_del_final));
+fprintf('ALL extractions done.\n');
+eval(sprintf('save %sCNMFE_BatchVer.mat %s -v7.3', outputdir, 'neuron_batch'));
+fprintf('ALL data saved, check them out!');

BatchVerMO/CTraceTestbed.m

@@ -0,0 +1,16 @@
+[AllC,boundary]=AllTraces(neuron_batch);
+%Boundary=boundary;
+
+figure; 
+plot(AllC(1,:))
+hold on
+plot(boundary'*ones(1,2), [0 max(AllC(1,:),[],2)], 'k:')
+
+    PartC{ni}=C;
+    boundary{ni}=cumsum(bound);
+
+
+figure; 
+plot(PartC{14})
+hold on
+plot(boundary{14}'*ones(1,2), [0 max(PartC{14},[],2)], 'k:')