diff --git a/.Rhistory b/.Rhistory new file mode 100644 index 0000000..e69de29 diff --git a/.gitignore b/.gitignore index 68ec758..c597cd3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +**.png + *.mat cvx/ *.avi diff --git a/BatchVerMO/A2C.m b/BatchVerMO/A2C.m new file mode 100644 index 0000000..15b7a5a --- /dev/null +++ b/BatchVerMO/A2C.m @@ -0,0 +1,110 @@ +function ACS_temp=A2C(Ysignal, A, options) +% ACS=A2C2A(ACS, File, A, options) +% General Description: +% This function is designed to extract C in background-subtracted signal +% from A(not Amask) in Ysignal. +% The method relies on CNMF-E's conference script. +% For each successful ai extraction, peel A*C off Ysignal. +% Iterate for all the neurons indicated by A. +% Input: +% Ysignal(background-subtracted), with dimenstion of [d1*d2,T]; +% A is also used for center calculation(for subtraction previous signal). +% options: struct data of paramters/options + % d1: number of rows + % d2: number of columns + % gSiz: maximum size of a neuron + % nb: left-over from the previous version, number of knots in + % modeling background,=1. + % min_corr: minimum threshold of correlation for segementing neurons + % deconv_options + % deconv_flag +% Output: +% ACS_temp is a structure with three fields: A,C,and C's +% standard deviation(STD). This structure is used for merging in later steps in cnmf_e-BatchVer. + +% Modified from "greedyROI_endoscope" +% Main dependences are "extract_a", "extract_c" in CNMF-E BatchVer, and "com" for neuron center by Pengcheng Zhou. + +% Shijie Gu, techel@live.cn + +%% parameters and preparations + +d1 = options.d1; % image height +d2 = options.d2; % image width +gSiz = options.gSiz; % average size of neurons + +Acenter = round(com(A, d1, d2)); %nr x 2 matrix, with the center of mass coordinates + +Ysignal(isnan(Ysignal)) = 0; % remove nan values +Ysignal = double(Ysignal); +T = size(Ysignal, 2); + +deconv_options_0= options.deconv_options; +deconv_flag = options.deconv_flag; + +K = size(A,2); +Cin = zeros(K, T); % temporal components +Cin_raw = zeros(K, T); % temporal components +STD = zeros(1,K); % standard deviation + +%% start initialization +for k = 1:K + r=Acenter(k,1); + c=Acenter(k,2); + + % Use center point data to roughly check whether this is a good starting point +% ind_p=sub2ind([d1 d2], r, c); +% y0 = Ysignal(ind_p, :); +% y0_std = std(diff(y0)); +% if max(diff(y0))< y0_std % signal is weak +% Ain(:,k)=A(:,k); % If it is a "poor" ci, no problem, low STD will let not it contribute much to finalA. +% Cin(k,:)=y0; % Since poor ci will get poor A that has bad shapes. Use normal A to fill in the place. +% STD(k)=std(y0); +% continue; +% end + + % extract ci + [ci_raw,ind_success_ci] = extract_c(Ysignal,[],A(:,k)); + Cin_raw(k,:)=ci_raw; + + if ~ind_success_ci % If it is a "poor" ci, no need to subtract it before next neuron. + Cin(k,:)=ci_raw; + STD(k)=std(ci_raw); + continue; + end + + if and(ind_success_ci,deconv_flag) + try + % deconv the temporal trace + [ci, ~, ~] = deconvolveCa(ci_raw, deconv_options_0); + % save this initialization + Cin(k, :) = ci; + STD(k)=std(ci); + ci=ci'; + catch + Cin(k, :)=ci_raw; + STD(k)=std(ci_raw); + ci=ci_raw; + end + end + + % select its neighbours to subtract some data, the box size is + %[2*gSiz+1, 2*gSiz+1] + % if the c is poor + ci_std = std(diff(ci)); + if max(diff(ci))< ci_std % signal is weak + continue + end + rsub = max(1, -gSiz+r):min(d1, gSiz+r); + csub = max(1, -gSiz+c):min(d2, gSiz+c); + [cind, rind] = meshgrid(csub, rsub); + ind_nhood = sub2ind([d1, d2], rind(:), cind(:)); + HY_box = Ysignal(ind_nhood, :); + Ysignal(ind_nhood, :) = HY_box - A(ind_nhood,k)*ci_raw; % update data +end +ACS_temp=struct('Cin',[],'Cin_raw',[],'STD',[]); +ACS_temp.Cin = Cin; +ACS_temp.Cin_raw=Cin_raw; +ACS_temp.STD = STD; + +end \ No newline at end of file diff --git a/BatchVerMO/A2image.m b/BatchVerMO/A2image.m new file mode 100644 index 0000000..265ab74 --- /dev/null +++ b/BatchVerMO/A2image.m @@ -0,0 +1,62 @@ +function image=A2image(A,d1,d2,textOrNot,color,PoporNOt,handle) + +if nargin<4 + textOrNot=false; +end + +if nargin<5 + color=[]; +end + +if nargin<6 + PoporNOt=false; +end + +A=A(:,any(A,1)); +k = size(A,2); +sA = sum(A,1); +A = bsxfun(@rdivide, A, sA)*prctile(sA, 5); +C=ones(k,1); + +if isempty(color)==false + if or(strcmp(color,'magenta'),strcmp(color,'red')) + color_palet = 1-[1 0 1]; + elseif strcmp(color,'green') + color_palet = 1-[0 1 0]; + end + nColors = repmat(color_palet,size(A,2),1); + Ir = diag(nColors(:,1)); + Ig = diag(nColors(:,2)); + Ib = diag(nColors(:,3)); + + Brainbow = 1-cat(3, A*Ir*C, A*Ig*C, A*Ib*C); + Brainbow = reshape(Brainbow,d1,d2,3); +else + Brainbow = 1-A*C; + Brainbow(Brainbow<0)=0; + Brainbow = reshape(Brainbow,d1,d2); + %image=imshow(Brainbow); +end +image=Brainbow; +if PoporNOt==true + if nargin<7 + figure + else + axes(handle) + end + imshow(Brainbow) +end + +if textOrNot==true + Atemp=reshape(A,d1,d2,k); + Position=zeros(2,k); + for i=1:k + Atemp_=Atemp(:,:,i); + [row_ind,col_ind] = find(Atemp_>0); + Position(2,i)=mean(row_ind); + Position(1,i)=mean(col_ind); + end + text(Position(1,:),Position(2,:),cellstr(num2str((1:k)'))','Color','black') + F = getframe(gca); + image=F.cdata; +end diff --git a/BatchVerMO/AllTraces.m b/BatchVerMO/AllTraces.m new file mode 100644 index 0000000..70fb24a --- /dev/null +++ b/BatchVerMO/AllTraces.m @@ -0,0 +1,27 @@ +function [AllC, boundary]=AllTraces(neuron_batch,type) +if nargin<2||isempty(type); + type='rawsignal'; +end + +AllC=[]; +boundary=[]; +if strcmp(type,'rawsignal') + for i=1:length(neuron_batch) + AllC=[AllC neuron_batch(i).rawsignal]; + boundary=[boundary size(neuron_batch(i).rawsignal,2)]; + end +elseif strcmp(type,'signal') + for i=1:length(neuron_batch) + AllC=[AllC neuron_batch(i).signal]; + boundary=[boundary size(neuron_batch(i).signal,2)]; + end +end +boundary=cumsum(boundary); +end +% %% +% framesize=[]; +% framesizeC=[]; +% for i=1:length(neuron_batch) +% framesize=[framesize size(neuron_batch(i).signal,2)]; +% framesizeC=[framesizeC size(neuron_batch(i).neuron.C,2)]; +% end \ No newline at end of file diff --git a/BatchVerMO/AllTracesfromACS.m b/BatchVerMO/AllTracesfromACS.m new file mode 100644 index 0000000..a763eee --- /dev/null +++ b/BatchVerMO/AllTracesfromACS.m @@ -0,0 +1,19 @@ +function [AllC, boundary]=AllTracesfromACS(ACS,rawOrNot) +if nargin<2 + rawOrNot=false; +end +AllC=[]; +boundary=[]; +for i=1:length(ACS) + if rawOrNot + AllC=[AllC ACS(i).Cin_raw]; + boundary=[boundary size(ACS(i).Cin_raw,2)]; + else + AllC=[AllC ACS(i).Cin]; + boundary=[boundary size(ACS(i).Cin,2)]; + end + boundary=cumsum(boundary); +end + + +end \ No newline at end of file diff --git a/BatchVerMO/BackgroundSub.m b/BatchVerMO/BackgroundSub.m new file mode 100644 index 0000000..747106f --- /dev/null +++ b/BatchVerMO/BackgroundSub.m @@ -0,0 +1,18 @@ +rr = ceil(neuron.options.gSiz * bg_neuron_ratio); +active_px = []; %(sum(IND, 2)>0); %If some missing neurons are not covered by active_px, use [] to replace IND +[Ybg, Ybg_weights] = neuron.localBG(Y, spatial_ds_factor, rr, active_px, neuron.P.sn, thresh); % estiamte local background. +% subtract the background from the raw data. +Ysignal = Y - Ybg; + +%% estimate noise +if ~isfield(neuron.P, 'sn') || isempty(neuron.P.sn) + %% estimate the noise for all pixels + b0 =zeros(size(Ysignal,1), 1); + sn = b0; + parfor m=1:size(neuron.A,1) + [b0(m), sn(m)] = estimate_baseline_noise(Ysignal(m, :)); + end + Ysignal = bsxfun(@minus, Ysignal, b0); + neuron.P.sn = sn; + %Ysignal = bsxfun(@minus, Ysignal, sn); +end \ No newline at end of file diff --git a/BatchVerMO/BatchVer.sh b/BatchVerMO/BatchVer.sh new file mode 100644 index 0000000..9e866e6 --- /dev/null +++ b/BatchVerMO/BatchVer.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +#SBATCH -n 1 +#SBATCH --cpus-per-task=8 +#SBATCH --mem=20000 +#SBATCH -t 0-2:00 +#SBATCH --time-min=0-01:00 +#SBATCH -o /net/feevault/data0/shared/EmilyShijieShared/BatchResultTest/job_%A.out +#SBATCH -e /net/feevault/data0/shared/EmilyShijieShared/BatchResultTest/job_%A.err + +cd /net/feevault/data0/shared/EmilyShijieShared/BatchResultTest/ +module add mit/matlab/2016b +matlab -nodisplay -singleCompThread -r "addpath(genpath('/home/shijiegu/cnmf_e/'));\ +BatchVerScript" \ No newline at end of file diff --git a/BatchVerMO/BatchVerSLURM.sh b/BatchVerMO/BatchVerSLURM.sh new file mode 100644 index 0000000..9a40f8f --- /dev/null +++ b/BatchVerMO/BatchVerSLURM.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +#SBATCH -n 1 +#SBATCH --cpus-per-task=8 +#SBATCH --mem=150000 +#SBATCH -t 0-10:00 +#SBATCH --time-min=0-01:00 +#SBATCH -o /net/feevault/data0/shared/EmilyShijieShared/BatchResult/6922FirstFewDaysForBatchNewMerging/job_%A.out +#SBATCH -e /net/feevault/data0/shared/EmilyShijieShared/BatchResult/6922FirstFewDaysForBatchNewMerging/job_%A.err +cd /net/feevault/data0/shared/EmilyShijieShared/BatchResult/6922FirstFewDaysForBatchNewMerging/ +module add mit/matlab/2016b +matlab -nodisplay -singleCompThread -r "addpath(genpath('/home/shijiegu/cnmf_e/'));\ +load(fullfile('/net/feevault/data0/shared/EmilyShijieShared/BatchResult/6922FirstFewDaysForBatchNewMerging/','PartOneOFcnmfeBatchVer.mat')); BatchVerTestVer " \ No newline at end of file diff --git a/BatchVerMO/BatchVerScript.m b/BatchVerMO/BatchVerScript.m new file mode 100644 index 0000000..55faf77 --- /dev/null +++ b/BatchVerMO/BatchVerScript.m @@ -0,0 +1,3 @@ +load(fullfile('/net/feevault/data0/shared/EmilyShijieShared/BatchResult/7030FirstFewDaysForBatch/','LogisticscnmfeBatchVer.mat')) +addpath(genpath(codeDir)); +cnmfeBatchVer_ClusterPart \ No newline at end of file diff --git a/BatchVerMO/BatchVerTestVer.m b/BatchVerMO/BatchVerTestVer.m new file mode 100644 index 0000000..568d285 --- /dev/null +++ b/BatchVerMO/BatchVerTestVer.m @@ -0,0 +1,77 @@ +%% 2 Merge similar neurons +%%% Merge similar neurons based on spatial AND temporal correlation +C_all=cat(2,ACS.Cin); + +Amask_temp=cat(2,A0s{1:2})>0; +C_temp=C_all(1:size(Amask_temp,2),:); +[Amask_temp,C_temp,ACS] = mergeAC(Amask_temp,C_temp,ACS,merge_thr); + +merge2start=1+size(A0s{1},2)+size(A0s{2},2); +for i=3:length(samplelist_reduced) + Amask_temp=cat(2,Amask_temp,A0s{i})>0; + + C_temp=[C_temp;C_all(merge2start:merge2start+size(A0s{i},2)-1,:)]; + + [Amask_temp,C_temp,ACS] = mergeAC(Amask_temp,C_temp,ACS,merge_thr); + merge2start=merge2start+size(A0s{i},2); +end + +save([outputdir 'commonAcnmfeBatchVer.mat'],'-v7.3') +%% 3 Determine uniqueA's +As=cell(1,length(samplelist)); +STDs=cell(1,length(samplelist)); +parfor i=1:length(samplelist) + As{i}=ACS(i).Ain; + STDs{i}=ACS(i).STD; +end +Afinal=ReducingA(As,STDs); +save([outputdir 'uniqueAcnmfeBatchVer.mat'],'-v7.3') +%% 5 Determine Afinal that will be used to extract C's in each file. + +%%% Some processes making Afinal nicer, modified from Pengcheng Zhou's +%%% idea. +for i=1:size(Afinal,2) + ai=Afinal(:,i); + temp = full(ai>quantile(ai, 0.5, 1)); + ai(~temp(:)) = 0; + Afinal(:,i)=ai; +end + +% Just in case some all zero A's got passed to this stage. +nz_ind=any(Afinal); +Afinal=Afinal(:,nz_ind); +save([outputdir 'AfinalcnmfeBatchVer.mat'],'-v7.3') +%% 6 "massive" procedure: Extract A from each file +neuron_batch(length(filelist)) = struct('ind_del',[],'signal',[],'FileOrigin',[],'neuron',[]); + +parfor i= 1:length(filelist) + mode='massive'; + nam=fullfile(datadir,filelist(i).name); + [~,neuron_batch(i)]=demo_endoscope2(gSig,gSiz,min_corr,min_pnr,FS,SSub,TSub,bg_neuron_ratio,nam,mode,[],Afinal,neuron_batch(i),convolveType); + neuron_batch(i).FileOrigin=filelist(i); % save origin(filelist) +end +fprintf('Massive extraction done.'); +save([outputdir 'MassivecnmfeBatchVer.mat'],'-v7.3') + +%neuron(length(filelist)) = struct('signal',[],'filelist',[]); +%%% Partition between those neurons found in each file and those not. +ind_del_final_cat=cat(2,neuron_batch.ind_del); +ind_del_final=any(ind_del_final_cat,2); +parfor i= 1:length(filelist) + neuron_batch(i).neuron.A=[neuron_batch(i).neuron.A(:,~ind_del_final) neuron_batch(i).neuron.A(:,ind_del_final)]; + fprintf('A extraction done\n'); + neuron_batch(i).neuron.C=[neuron_batch(i).neuron.C(~ind_del_final,:);neuron_batch(i).neuron.C(ind_del_final,:)]; + neuron_batch(i).neuron.C_raw=[neuron_batch(i).neuron.C_raw(~ind_del_final,:);neuron_batch(i).neuron.C_raw(ind_del_final,:)]; + fprintf('C extraction done\n'); + %%% save each data i's signal into neuron(i).signal and + for j=1:size(neuron_batch(i).neuron.A,2) + jA=neuron_batch(i).neuron.A(:,j); + jC=neuron_batch(i).neuron.C(j,:); + neuron_batch(i).signal(j,:)=median(jA(jA>0)*jC); + end + fprintf('neuron_batch %.0f extraction done\n', i); +end +fprintf('First %.0f neurons are successfully deconvolved in each files while those after that are missing in some files\n', sum(~ind_del_final)); +fprintf('ALL extractions done.\n'); +eval(sprintf('save %sCNMFE_BatchVer.mat %s -v7.3', outputdir, 'neuron_batch')); +fprintf('ALL data saved, check them out!'); \ No newline at end of file diff --git a/BatchVerMO/CTraceTestbed.m b/BatchVerMO/CTraceTestbed.m new file mode 100644 index 0000000..412e2d7 --- /dev/null +++ b/BatchVerMO/CTraceTestbed.m @@ -0,0 +1,16 @@ +[AllC,boundary]=AllTraces(neuron_batch); +%Boundary=boundary; + +figure; +plot(AllC(1,:)) +hold on +plot(boundary'*ones(1,2), [0 max(AllC(1,:),[],2)], 'k:') + + PartC{ni}=C; + boundary{ni}=cumsum(bound); + + +figure; +plot(PartC{14}) +hold on +plot(boundary{14}'*ones(1,2), [0 max(PartC{14},[],2)], 'k:') \ No newline at end of file diff --git a/BatchVerMO/ColorAllNeurons.m b/BatchVerMO/ColorAllNeurons.m new file mode 100644 index 0000000..733770c --- /dev/null +++ b/BatchVerMO/ColorAllNeurons.m @@ -0,0 +1,49 @@ +function Brainbow=ColorAllNeurons(A,d1,d2,Picname,outputdir) + if nargin <5 + outputdir = {}; + end + if nargin<4 + Picname = {}; + end + +% figure; + hold all + color_palet = 1-[[1 0 0]; [1 .6 0]; [.7 .6 .4]; [.6 .8 .3]; [0 .6 .3]; [0 0 1]; [0 .6 1]; [0 .7 .7]; [.7 0 .7]; [.7 .4 1]]; + color_palet = color_palet([1:2:end 2:2:end],:); % scramble slightly + nColors = color_palet(mod(1:(size(A,2)),size(color_palet,1))+1,:); + Ir = diag(nColors(:,1)); + Ig = diag(nColors(:,2)); + Ib = diag(nColors(:,3)); + + + k = size(A,2); + sA = sum(A,1); + A = bsxfun(@rdivide, A, sA)*prctile(sA, 5); + + C=ones(k,1); + + Brainbow = 1-cat(3, A*Ir*C, A*Ig*C, A*Ib*C); + Brainbow = reshape(Brainbow,d1,d2,3); + image(Brainbow) + set(gca, 'ydir', 'reverse') + axis image; axis off; shg + + Atemp=reshape(A,d1,d2,k); + Position=zeros(2,k); + for i=1:k + Atemp_=Atemp(:,:,i); + [row_ind,col_ind] = find(Atemp_>0); + Position(2,i)=mean(row_ind); + Position(1,i)=mean(col_ind); + end + text(Position(1,:),Position(2,:),cellstr(num2str((1:k)'))','Color','black') + + fig=gcf; + if numel(Picname)>0 + title(Picname,'interpreter','none') + + fignam=[outputdir Picname,'.png']; + + saveas(gcf,fignam); + end + \ No newline at end of file diff --git a/BatchVerMO/ColorAllNeuronsForMo.m b/BatchVerMO/ColorAllNeuronsForMo.m new file mode 100644 index 0000000..ef70e9a --- /dev/null +++ b/BatchVerMO/ColorAllNeuronsForMo.m @@ -0,0 +1,58 @@ +function [xx_s_e,xx_f_e,yy_s_e,yy_f_e]=ColorAllNeuronsForMo(A,d1,d2,Picname,outputdir,xxsfyysf,overlap) + % with grid plotting + xx_s=xxsfyysf{1}; + xx_f=xxsfyysf{2}; + yy_s=xxsfyysf{3}; + yy_f=xxsfyysf{4}; + xx_s_e=[]; xx_f_e=[]; yy_s_e=[]; yy_f_e=[]; + for i=1:length(xx_s) + for j=1:length(yy_s) + xx_s_e=[xx_s_e max(xx_s(i)-overlap(1),xx_s(1))]; + xx_f_e=[xx_f_e min(xx_f(i)+overlap(1),xx_f(end))]; + yy_s_e=[yy_s_e max(yy_s(j)-overlap(2),yy_s(1))]; + yy_f_e=[yy_f_e min(yy_f(j)+overlap(2),yy_f(end))]; + end + end + + + figure; + hold all + + color_palet = 1-[[1 0 0]; [1 .6 0]; [.7 .6 .4]; [.6 .8 .3]; [0 .6 .3]; [0 0 1]; [0 .6 1]; [0 .7 .7]; [.7 0 .7]; [.7 .4 1]]; + color_palet = color_palet([1:2:end 2:2:end],:); % scramble slightly + nColors = color_palet(mod(1:(size(A,2)),size(color_palet,1))+1,:); + Ir = diag(nColors(:,1)); + Ig = diag(nColors(:,2)); + Ib = diag(nColors(:,3)); + + + k = size(A,2); + sA = sum(A,1); + A = bsxfun(@rdivide, A, sA)*prctile(sA, 5); + + C=ones(k,1); + + Brainbow = 1-cat(3, A*Ir*C, A*Ig*C, A*Ib*C); + Brainbow = reshape(Brainbow,d1,d2,3); + image(Brainbow) + set(gca, 'ydir', 'reverse') + axis image; axis off; shg + + Atemp=reshape(A,d1,d2,k); + Position=zeros(2,k); + for i=1:k + Atemp_=Atemp(:,:,i); + [row_ind,col_ind] = find(Atemp_>0); + Position(2,i)=mean(row_ind); + Position(1,i)=mean(col_ind); + end + + plot([0 d2], [xx_s_e'; xx_f_e']*[1 1], ':', 'color', .7*[1 1 1]); + plot([yy_s_e'; yy_f_e']*[1 1],[0 d1], ':', 'color', .7*[1 1 1]); + + text(Position(1,:),Position(2,:),cellstr(num2str((1:k)'))','Color','black') + title(Picname,'interpreter','none') + + fignam=[outputdir Picname,'.png']; + saveas(gcf,fignam); + \ No newline at end of file diff --git a/BatchVerMO/DEMO.m b/BatchVerMO/DEMO.m new file mode 100644 index 0000000..71cd63c --- /dev/null +++ b/BatchVerMO/DEMO.m @@ -0,0 +1,98 @@ +%% Demo for cnmfe (BatchVer) +% modified by Shijie Gu and Emily Mackevicius + +%% A. Input on your PC +% (1) code directory, data directory, sample directory, what to sample, what to extract. +% (2) normal CNMF-E parameters. +% (3) other parameters. +clear all +%(0) +%codedir=... +%addpath(genpath(codedir)); +codeDir='/home/shijiegu/cnmf_e/'; % codeDir2='/home/shijiegu/caprocessing/'; + +%(1) Specify where data are on your local machine. This is a parsing +%step. + +[datadir,sampledir,outputdir,filelist,samplelist]=... + InputOutput('datadir','/Volumes/shared/EmilyShijieShared/ProcessedCalciumData/6991FirstFewDaysForBatch/',... + 'sampledir',[],... + 'outputdir','/Volumes/shared/EmilyShijieShared/ProcessedCalciumData/6991FirstFewDaysForBatch/Singing-BatchVerResult-071607170718/',... + 'datakind','*Ca*',... + 'samplekind','*',... + 'SamplingMethod','auto',... + 'dataMethod','manual'); +%(1.1, replace for actual dir on cluster) +datadir=strrep(datadir,'/Volumes/shared/','/net/feevault/data0/shared/'); +sampledir=strrep(sampledir,'/Volumes/shared/','/net/feevault/data0/shared/'); +outputdir_local=outputdir; +outputdir=strrep(outputdir,'/Volumes/shared/','/net/feevault/data0/shared/'); +if ~exist(outputdir_local,'dir') + mkdir(outputdir_local) +end +daynum=201704270501; +Aoutputdir=outputdir; + +%(2) +gSig=10; +gSiz=17; +min_corr=0.75; +min_pnr=6; +min_pixel = 50; % minimum number of nonzero pixels for each neuron +bd = 1; % number of rows/columns to be ignored in the boundary (mainly for motion corrected data) + +bg_neuron_ratio = 1; % spatial range / diameter of neurons +FS=30; +SSub=1; +TSub=1; +convolveType='ar1'; % convolveType: string, defines the model of the deconvolution kernel. possible options are: + % 'ar1': auto-regressive model with order p=1 + % 'ar2': auto-regressive model with order p=2 + % 'exp2': the convolution kernel is modeled as the difference of two + % exponential functions - + % h(t) = (exp(-t/tau_d) - exp(-t/tau_r)) / (tau_d-tau_r) + % 'kernel': a vector of the convolution kernel +merge_thr=[0.7,0.5,0]; +%namepattern=:; % In sampling stage, for each file running cnmfe, save A's so you can roughly check what neuron is picked in which file. + % Each pic's name is from some characters from raw + % data's filename. Here I use 1:35 +%(3) +% parameters for putting similar neurons in sequence. +correlation_thresh=0.6; % Those neuron pairs with cross correlation coefficient above + % correlation_thresh will be considered similar, and + % will be put in similar sequence in initiation. +max2max2nd=1.1; % type help Over_Days_findAnn +skewnessthresh=0; % type help findn1n2 +% Merge similar neurons based on spatial AND temporal correlation +merge_thr_2=[0.7,0.5]; + +%(4) +running_on_cluster=true; +workersnum=4; + +% Finally, save all these parameters/variables into LogisticscnmfeBatchVer.mat +ininame='LogisticscnmfeBatchVer.mat'; +save([outputdir_local ininame]) + +%% B. making cluster script +fileID = fopen('BatchVerSLURM6991Sing161718.sh','w'); +request={'#!/bin/bash' + '' + '#SBATCH -n 1' + '#SBATCH --cpus-per-task=8' + '#SBATCH --mem=200000' + '#SBATCH -t 0-10:00' + '#SBATCH --time-min=0-01:00'}; +fprintf(fileID,'%s\n',request{:}); + +errorANDoutAndcd= ['#SBATCH -o %sjob_%%A.out\n#SBATCH -e %sjob_%%A.err\n',... + 'cd %s\n']; +fprintf(fileID,errorANDoutAndcd,outputdir,outputdir,outputdir); + +MATLABcommands=['module add mit/matlab/2016b\n',... + 'matlab -nodisplay -singleCompThread -r "addpath(genpath(''%s''));\\\n',... + 'load(fullfile(''%s'',''%s'')); cnmfeBatchVer_ClusterPart"']; +fprintf(fileID,MATLABcommands,codeDir,outputdir,ininame); + +fclose(fileID); +%chmod g+rwx /net/feevault/data0/shared/EmilyShijieShared diff --git a/BatchVerMO/DEMO.m.orig b/BatchVerMO/DEMO.m.orig new file mode 100644 index 0000000..4ab490e --- /dev/null +++ b/BatchVerMO/DEMO.m.orig @@ -0,0 +1,98 @@ +%% Demo for cnmfe (BatchVer) +% Shijie Gu and Emily Mackevicius + +%% A. Input on your PC +% (1) code directory, data directory, sample directory, what to sample, what to extract. +% (2) normal CNMF-E parameters. +% (3) other parameters. + +%(0) +clear all +codedir='C:\Users\emackev\Documents\MATLAB\cnmf_e\BatchVer'; % local computer code path +addpath(genpath(codedir)); +codeDir='/home/elm/CaImagingCode/cnmf_e/'; % cluster code path + +%(1) Specify where data is on your local machine. This is a parsing +%step. +[datadir,sampledir,outputdir,filelist,samplelist]=... +<<<<<<< HEAD + InputOutput('datadir','/Volumes/data0-shared/elm/ProcessedCalciumData/7030FirstFewDaysForBatch/',... + 'sampledir',[],... + 'outputdir','/Volumes/shared/EmilyShijieShared/BatchResult/6938FirstFewDaysForBatch/',... + 'datakind','*CaELM*',... + 'samplekind','*CaELM*',... +======= + InputOutput('datadir','//feevault/data0/ProcessedCalciumData/6922FirstFewDaysForBatch/JustSinging',... % data directory on local computer + 'sampledir','//feevault/data0/ProcessedCalciumData/6922FirstFewDaysForBatch/JustSinging',... % directory of sample (seed) examples, if necessary + 'outputdir','//feevault/shared/EmilyShijieShared/BatchResult/6922FirstFewDaysForBatch/',... + 'datakind','*CaELM*',... % reg exp for data files + 'samplekind','*CaELM*',... % reg exp for sample (seed) examples +>>>>>>> 29ea0866ca7adee5c4a78abf47ac9e0c19db4fe5 + 'SamplingMethod','auto'); +%(1.1, replace for actual dir on cluster) +datadir=strrep(datadir,'\\feevault\data0\','/net/feevault/data0/elm/'); +datadir=strrep(datadir,'\','/'); +sampledir=strrep(sampledir,'\\feevault\data0\','/net/feevault/data0/elm/'); +sampledir=strrep(sampledir,'\','/'); +outputdir_local=outputdir; +outputdir=strrep(outputdir,'//feevault/','/net/feevault/data0/elm/'); +sampledir=strrep(sampledir,'\','/'); +mkdir(outputdir_local) + +%(2) +gSig=10; +gSiz=17; +min_corr=0.85; +min_pnr=6.5; +bg_neuron_ratio = 1; % spatial range / diameter of neurons +FS=30; +SSub=1; +TSub=1; +convolveType='ar1'; % convolveType: string, defines the model of the deconvolution kernel. possible options are: + % 'ar1': auto-regressive model with order p=1 + % 'ar2': auto-regressive model with order p=2 + % 'exp2': the convolution kernel is modeled as the difference of two + % exponential functions - + % h(t) = (exp(-t/tau_d) - exp(-t/tau_r)) / (tau_d-tau_r) + % 'kernel': a vector of the convolution kernel +namepattern=1:35; % In sampling stage, for each file running cnmfe, save A's so you can roughly check what neuron is picked in which file. + % Each pic's name is from some characters from raw + % data's filename. Here I use 1:35 +%(3) +% parameters for putting similar neurons in sequence. +correlation_thresh=0.6; % Those neuron pairs with cross correlation coefficient above + % correlation_thresh will be considered similar, and + % will be put in similar sequence in initiation. +max2max2nd=1.1; % type help Over_Days_findAnn +skewnessthresh=0; % type help findn1n2 +% Merge similar neurons based on spatial AND temporal correlation +merge_thr=[0.7,0.7]; + +%(4) +running_on_cluster=true; +workersnum=4; + +% Finally, save all these parameters/variables into LogisticscnmfeBatchVer.mat +ininame='LogisticscnmfeBatchVer.mat'; +save([outputdir_local ininame]) + +%% B. making cluster script +fileID = fopen(fullfile(codedir, 'BatchVerSLURM.sh'),'w'); +request={'#!/bin/bash' + '#SBATCH -n 1' + '#SBATCH --cpus-per-task=8' + '#SBATCH --mem=100000' + '#SBATCH -t 0-10:00' + '#SBATCH --time-min=0-01:00'}; +fprintf(fileID,'%s\n',request{:}); + +errorANDoutAndcd= ['#SBATCH -o %sjob_%%A.out\n#SBATCH -e %sjob_%%A.err\n',... + 'cd %s\n']; +fprintf(fileID,errorANDoutAndcd,outputdir,outputdir,outputdir); + +MATLABcommands=['module add mit/matlab/2016b\n',... + 'matlab -nodisplay -singleCompThread -r "addpath(genpath(''%s''));\\\n',... + 'load(fullfile(''%s'',''%s'')); cnmfeBatchVer_ClusterPart "']; +fprintf(fileID,MATLABcommands,codeDir,outputdir,ininame); + +fclose(fileID); diff --git a/BatchVerMO/DEMOforMoBatchVer.m b/BatchVerMO/DEMOforMoBatchVer.m new file mode 100644 index 0000000..2fe67d0 --- /dev/null +++ b/BatchVerMO/DEMOforMoBatchVer.m @@ -0,0 +1,203 @@ +%% Demo for cnmfe (BatchVerMO) +% modified by Shijie Gu and Emily Mackevicius + +%% A. Input on your PC +% I normal CNMF-E parameters and other new parameters for CNMF-E. +% II 1/code directory... [codeDir] +% 2/output directory...[outputdir,outputdirDetails,datadir,filelist,samplelist] +% 3/data directory...sample directory, what to sample, what to extract. +% 4/days [totaldays] + +clear all +% I PARAMETERS +% ----(2) +% about loading data +sframe=1; % first frame to read (optional, default:1) +num2read=[]; % how many frames to read (optional, default: until the end) + +% create Source2D class object for storing results and parameters +with_dendrites=false; % with dendrites or not +K = []; % maximum number of neurons to search. + % Use [] to search the number automatically +neuron_full = Sources2D('ssub',1, 'tsub',1, ... % downsampling + 'gSig',10,... % sigma of the 2D gaussian (width of the gaussian kernel) that approximates cell bodies + 'gSiz',17,... % maximum diameter of neurons in the image plane. larger values are preferred. + 'min_corr',0.8,'min_pnr',14, ... + 'min_pixel',50,... % minimum number of nonzero pixels for each neuron + 'bd',1,... % number of rows/columns to be ignored in the boundary (mainly for motion corrected data) + 'center_psf',true); +neuron_full.Fs = 30; % frame rate +% options for running deconvolution +neuron_full.options.deconv_options = struct('type', 'ar1', ... % model of the calcium traces. {'ar1', 'ar2'} + ...% convolveType: string, defines the model of the deconvolution kernel. possible options are: + ... % 'ar1': auto-regressive model with order p=1 + ... % 'ar2': auto-regressive model with order p=2 + ... % 'exp2': the convolution kernel is modeled as the difference of two + ... % exponential functions - + ... % h(t) = (exp(-t/tau_d) - exp(-t/tau_r)) / (tau_d-tau_r) + ... % 'kernel': a vector of the convolution kernel + 'method', 'constrained', ... % method for running deconvolution {'foopsi', 'constrained', 'thresholded'} + 'optimize_pars', true, ... % optimize AR coefficients + 'optimize_b', true, ... % optimize the baseline + 'optimize_smin', true); % optimize the threshold + +% Parameters in updating +bg_neuron_ratio = 1; % spatial range / diameter of neurons + +% parameters for putting similar neurons in sequence. +correlation_thresh=0.6; % Those neuron pairs with cross correlation coefficient above + % correlation_thresh will be considered similar, and + % will be put in similar sequence in initiation. +max2max2nd=1.1; % type help Over_Days_findAnn +skewnessthresh=0; % type help findn1n2 + +% Merge similar neurons based on spatial AND temporal correlation +% "merge_thr_2" here will be used in MergeAC, the BatchVer specific merging, +% while "merge_thr" in section2 is used for for normal CNMF-E neuron merging in initiation and iteration process. +merge_thr=[0.7,0.5,0]; +merge_thr_2=[0.7,0.5]; + +% Merge neurons that are very close. +dmin=5; + +thresh_detecting_frames=10; % threshold for detecting frames with large cellular activity. (mean of neighbors' activity + thresh*sn) + +%%%% There are a few more techinical parametrs in demo_endoscope2. + +% Cluster-specific parameters, depend on the cluster you use and the CPU number you request. + % With this workersnum, cnmfeBatchVer_ClusterPart has functions written to create a cluster and a parpool within it. + % The design caters to parpool in cluster use. The functions for these are in a seperate folder "HandlingParforbyGalen" which is written by Galen Lynch. +running_on_cluster=true; +workersnum=4; + +% II code directory and output directory + +Version='BatchVer'; %with motion: MoBatchVer; without motion: BatchVer. +% ---- /1/---------------------------------------------------------------- +codeDir='/home/shijiegu/cnmf_e/'; %where code is on the cluster +% ------------------------------------------------------------------------ + +% ---- /2/Output directory, local and on cluster-------------------------- +outputdir_local='/Volumes/shared/EmilyShijieShared/CaExtraction/6938JustSinging/'; +outputdir='/net/feevault/data0/shared/EmilyShijieShared/CaExtraction/6938JustSinging/'; +%outputdir=strrep(outputdir_local,'/Volumes/data0-shared/elm/ProcessedCalciumData/','/net/feevault/data0/shared/EmilyShijieShared/CaExtraction/6701/'); +% ------------------------------------------------------------------------ + +% -----------------------------ignore this auto area---------------------- +if ~exist(outputdir_local,'dir') + mkdir(outputdir_local) +end +% ------------------------------------------------------------------------ + +% ---- /3/ "Motion batch", within batch, no motion, between batch there can be motion.- +% If using BatchVer, you can just input one element in totaldays. +%totaldays=6938; +totaldays_act=[6938]; +totaldays=6938; %[0612 0613 0614 0615 0617 0618 0620 0621]; +% ------------------------------------------------------------------------- + +% -----/4/ Fill in datadir on local machine, datakind---------------------- +for i=1:length(totaldays_act) + daynum=totaldays_act(i); + if strcmp(Version,'MoBatchVer') + outputdirDetails = [outputdir, num2str(totaldays_act(i)), '/']; + elseif strcmp(Version,'BatchVer') + outputdirDetails = [outputdir, 'details', '/']; + end + + [datadir,sampledir,~,filelist,samplelist]=... + InputOutput('datadir','/Volumes/shared/EmilyShijieShared/ProcessedCalciumData/6938FirstFewDaysForBatch/Singing/',... + 'datakind',['*' num2str(totaldays_act(i)) '*'],... + 'DataMethod','auto'); + % if there is any motion within a day + %Add another day in totaldays, use ('DataMethod','manual'); + % replace for actual dir on cluster + datadir=strrep(datadir,'/Volumes/','/net/feevault/data0/'); + sampledir=strrep(sampledir,'/Volumes/','/net/feevault/data0/'); + + % Finally, save all these parameters/variables into LogisticscnmfeBatchVer.mat + ininame='LogisticscnmfeBatchVer'; + save([outputdir_local ininame num2str(totaldays_act(i)) '.mat']) +end +% ---------------------------------------------------------------------------------------- + +%% B. making cluster script +% To better control each day's cnmfe behavior, each day has its own script, +% thus constinuting one job-one day. For all these scripts, use MATLAB to +% write in jobdir folder with the common prefix as variable "shellname". + +% (1) jobdir folder, shellname +% ---- /1/ ---------------------------------------------------------------- +jobdir='BatchVer6938'; +local_jobdir=['/Users/gushijie/Documents/MATLAB/Jobs/' jobdir '/']; +shellname=jobdir; +% ------------------------------------------------------------------------- + +% -----------------------------ignore this auto area----------------------- +if ~exist(local_jobdir,'dir') + mkdir(local_jobdir) +end +cd(local_jobdir) +% ------------------------------------------------------------------------- + +for i=1:length(totaldays) + scriptname=sprintf([shellname '_%.0f.sbatch'],totaldays(i)); + fileID = fopen(scriptname,'w'); + request=['#!/bin/bash\n',... + '\n',... + '#SBATCH -n 1\n',... + '#SBATCH --cpus-per-task=8\n',... + '#SBATCH --mem=200000\n',... + '#SBATCH -t 0-10:00\n',... + '#SBATCH --time-min=0-01:00\n']; + fprintf(fileID, request); + + errorANDoutAndcd= ['#SBATCH -o %sjob_%%A_%s.out\n',... + '#SBATCH -e %sjob_%%A_%s.err\n',... + '\n']; + fprintf(fileID,errorANDoutAndcd,outputdir,num2str(totaldays(i)),outputdir,num2str(totaldays(i))); + + MATLABcommands=['module add mit/matlab/2016b\n',... + 'matlab -nodisplay -singleCompThread -r "addpath(genpath(''%s''));\\\n',... + 'load(''%s''); cnmfeMoBatchVer_ClusterPart"']; + fprintf(fileID,MATLABcommands,codeDir,[outputdir,'LogisticscnmfeBatchVer',num2str(totaldays(i)),'.mat']); + + fclose(fileID); +end +%chmod g+rwx /net/feevault/data0/shared/EmilyShijieShared + +%% B2. making the shell script +fileID = fopen([shellname '.sh'],'w'); +forloop=['cd %s\n',... + 'for file in %s; do\n',... + 'sbatch %s_${file}.sbatch\n',... + 'sleep 1\n',... + 'done']; +fprintf(fileID,forloop,jobdir,num2str(totaldays),shellname); + +%% B3. (MoBatchVer only) for cnmfeMoBatchVer_ClusterPart2 +scriptname=sprintf([shellname '_part2.sbatch']); +fileID = fopen(scriptname,'w'); +request=['#!/bin/bash\n',... + '\n',... + '#SBATCH -n 1\n',... + '#SBATCH --cpus-per-task=8\n',... + '#SBATCH --mem=200000\n',... + '#SBATCH -t 0-20:00\n',... + '#SBATCH --time-min=0-01:00\n']; +fprintf(fileID, request); + +errorANDoutAndcd= ['#SBATCH -o %sjob_%%A.out\n',... + '#SBATCH -e %sjob_%%A.err\n',... + '\n']; +fprintf(fileID,errorANDoutAndcd,outputdir,outputdir); + +MATLABcommands=['cd %s\n',... + 'module add mit/matlab/2016b\n',... + 'matlab -nodisplay -singleCompThread -r "addpath(genpath(''%s''));addpath(genpath(''/home/shijiegu/caprocessing/''));\\\n',... + 'outputdir=''%s'';',... + 'load(fullfile(outputdir,''%s''));load(fullfile(outputdir,''cnmfe_BatchVer_PartII_MotionCorrection.mat''));\\\n',... + 'cnmfeMoBatchVer_ClusterPart2"']; +fprintf(fileID,MATLABcommands,outputdir,codeDir,outputdir,['LogisticscnmfeBatchVer',num2str(totaldays(1)),'.mat']); + +fclose(fileID); diff --git a/BatchVerMO/InputOutput.m b/BatchVerMO/InputOutput.m new file mode 100644 index 0000000..4bab784 --- /dev/null +++ b/BatchVerMO/InputOutput.m @@ -0,0 +1,122 @@ +function [datadir,sampledir,outputdir,filelist,samplelist]=InputOutput(varargin) +% [datadir,sampledir,outputdir,filelist,samplelist]=InputOutput(varargin) +% Example: [datadir,sampledir,outputdir,filelist,samplelist]= +% InputOutput('datadir','/Volumes/data0-shared/elm/ProcessedCalciumData/7030FirstFewDaysForBatch/JustAFewToTest/', +% 'datakind','*CaELM*', +% 'DataMethod','manual'); +% Input: 1-data directory, 2-sample directory, which can be empty. If +% 'SamplingMethod' is 'auto', the sample directory will be the same as data directory +% 3-output directory, 4/5-what "kind" of data you want to read in to extract cells (for data and sample). +% 6/7-how to choose data for sampling or extraction, either "auto", every some files will be asked for you +% as an input to sample data, whereas in choosing all data for extraction, "auto" automatically choose all data; +% or, "manual", a window will pop up for you to choose data from the sample/data dir you input. +% More notes: +% "kind" is wildcard in matlab. This is used in dir() function, see +% documentation dir() for how to use this well. +% When choosing files "manual"ly, select multiple files by +% holding down the Shift or Ctrl key and clicking on additional file names. +% IMPORTANT: All samples must come from one folder. + +% Shijie Gu, techel@live.cn + +p = inputParser; +addParameter(p,'datadir',[]); +addParameter(p,'sampledir',[]); +addParameter(p,'outputdir',[]); +kinddefault='*'; %read in everything in the folder. +addParameter(p,'datakind',kinddefault); +addParameter(p,'samplekind',kinddefault); +addParameter(p,'every_file_num',1); % left-over from previous design, do not change this parameter. +expectedmethod = {'auto','manual'}; +addParameter(p,'SamplingMethod','auto', @(x) any(validatestring(x,expectedmethod))); % left-over from previous design, do not change this parameter. +addParameter(p,'DataMethod','auto', @(x) any(validatestring(x,expectedmethod))); + +parse(p, varargin{:}); + +datadir=p.Results.datadir; +sampledir=p.Results.sampledir; +outputdir=p.Results.outputdir; +datakind=p.Results.datakind; +samplekind=p.Results.samplekind; +every_file_num=p.Results.every_file_num; +SamplingMethod=p.Results.SamplingMethod; +DataMethod=p.Results.DataMethod; +if strcmp(samplekind,'*') + samplekind = datakind; +end + +if strcmp(DataMethod,'auto') + Datadir=fullfile(datadir,datakind); + filelist=dir(Datadir); + numInFolder=numel(filelist); + %first check how many input data files are there. Minimum requirement is two. + if numInFolder==0 + ME = MException('cnmfeBatchVer:NotEnoughInput', 'This is an empty data folder or a incorrectly specified folder. \n Please redefine input.'); + throw(ME) + elseif numInFolder==1 + ME2 = MException('cnmfeBatchVer:OnlyOneInput', 'Only one file \n Use normal cnmf-e instead.'); + throw(ME2) + end + fprintf('Chosen %.0f files to extract in the end. \n', numInFolder); +else + [FileName,PathName,~] = uigetfile({'*.*';datakind},'cnmfe(BatchVer)-Manually Choose Data',datadir,'MultiSelect','on'); + datadir=PathName; %Just in case in practice, user changed directory in the folder. + numInFolder=numel(FileName); + %first check how many input data files are there. Minimum requirement is two. + if numInFolder==0 + ME = MException('cnmfeBatchVer:NotEnoughInput', 'This is an empty data folder or a incorrectly specified folder. \n Please redefine input.'); + throw(ME) + elseif numInFolder==1 + ME2 = MException('cnmfeBatchVer:OnlyOneInput', 'Only one file \n Use normal cnmf-e instead.'); + throw(ME2) + end + fprintf('Chosen %.0f files to extract in the end. \n', numInFolder); + for i=1:length(FileName) + filelist(i)= dir(fullfile(PathName,FileName{i})); + end +end + +if isempty(sampledir) + sampledir=datadir; +end + +if strcmp(SamplingMethod,'auto') + samplelistfull=filelist; + sampleInFolder=numel(samplelistfull); + if isempty(every_file_num) + % choose final A from every every_file_num as samples in the folders. + prompt=sprintf('%.0f files in the sample folder, \n input x so that every x files to sample as input. Input x then hit "enter": ', sampleInFolder); + every_file_num = input(prompt); + end +% if isempty(every_file_num) +% every_file_num=max(1,sampleInFolder/10); +% end + + if every_file_num==1 + choose_ind=true(numel(samplelistfull),1); + else + choose_ind=mod(1:numel(samplelistfull),every_file_num)==1; + end + + if numel(samplelistfull)<=every_file_num + error('Zero file for cnmf-e BatchVer sampling. Please lower your every_file_num') + elseif numel(samplelistfull)<2*every_file_num + warning('Only one file for cnmf-e BatchVer sampling. Lowering your every_file_num is suggested.') + elseif sum(choose_ind)<5 + warning('less than 5 files selected as sample for data. Be aware of this.') + end + samplelist=samplelistfull(choose_ind); +else + [FileName,PathName,~] = uigetfile({'*.*';samplekind},'cnmfe(BatchVer)-Manually Choose Sample',sampledir,'MultiSelect','on'); + sprintf('Chosen %.0f files as samples. \n', length(FileName)); + if numel(FileName)==1 + warning('Only one file for cnmf-e BatchVer sampling. Lowering your every_file_num is suggested.') + elseif numel(FileName)<5 + warning('less than 5 files selected as sample for data. Be aware of this.') + end + sampledir=PathName; %Just in case in practice, user changed directory in the folder. + samplelist = struct('name',FileName); +end + +end + diff --git a/BatchVerMO/MakingVideos.m b/BatchVerMO/MakingVideos.m new file mode 100644 index 0000000..63886dd --- /dev/null +++ b/BatchVerMO/MakingVideos.m @@ -0,0 +1,144 @@ +function [Ysignal,Ybg,Yac,center_ac]=MakingVideos(File,neuron_batchMO,str_p,datadir,filelist,d1,d2,currentday,outputdir,kt) +%% Making videos for each day's data +% kt: avi_file.FrameRate= neuron.Fs/kt; + +% some parameters for default +t_begin = 1; + +Ysignal=[]; +Yac=[]; +Ybg=[]; +center_ac=[]; + +for i=1:length(File) + Ysignal=[Ysignal,File(i).Ysignal]; + + Ybg=[Ybg,File(i).Ybg]; + display(size(Ybg)) + + j=i+str_p; + Yac=[Yac,neuron_batchMO(j).neuron.A*neuron_batchMO(j).neuron.C]; + center_ac=[center_ac; max(neuron_batchMO(j).neuron.A,[],1)'.*max(neuron_batchMO(j).neuron.C,[],2)]; + + display(size(Yac)) +end +Ysignal=reshape(Ysignal,d1,d2,[]); +Yac=reshape(Yac,d1,d2,[]); +Ybg=reshape(Ybg,d1,d2,[]); +center_ac=median(center_ac); +t_end=size(Yac,3); +display(t_end) +display(size(Ybg)) + +Y=[]; +for i=1:length(filelist) + Y_tmp=matfile(fullfile(datadir,filelist(i).name)); + Y=cat(3,Y,Y_tmp.Y); +end +Y=double(Y); + +figure('position', [0,0, 600, 400]); + + +range_res = [-1,1]*center_ac; +if ~exist('range_ac', 'var') + range_ac = center_ac*1.01+range_res; +end + +if ~exist('range_Y', 'var') + if ~exist('multi_factor', 'var') + temp = quantile(Y(randi(numel(Y), 10000,1)), [0.01, 0.98]); + multi_factor = ceil(diff(temp)/diff(range_ac)); + else + temp = quantile(Y(randi(numel(Y), 10000,1)), 0.01); + end + center_Y = temp(1) + multi_factor*center_ac; + range_Y = center_Y + range_res*multi_factor; +end +%% create avi file + +avi_file = VideoWriter([outputdir currentday '_videos.avi']); +if ~isnan(File(1).neuron.Fs) + avi_file.FrameRate= File(1).neuron.Fs/kt; +end +avi_file.open(); + +ax_y = axes('position', [0.015, 0.51, 0.3, 0.42]); +ax_bg= axes('position', [0.015, 0.01, 0.3, 0.42]); +ax_signal= axes('position', [0.345, 0.51, 0.3, 0.42]); +ax_denoised = axes('position', [0.345, 0.01, 0.3, 0.42]); +ax_res = axes('position', [0.675, 0.51, 0.3, 0.42]); + +for m=t_begin:kt:t_end + axes(ax_y); cla; + imagesc(Y(:, :,m), range_Y); + % set(gca, 'children', flipud(get(gca, 'children'))); + title('Raw data'); + axis equal off tight; + + axes(ax_bg); cla; + imagesc(Ybg(:, :, m),range_Y); + % set(gca, 'children', flipud(get(gca, 'children'))); + axis equal off tight; + title('Background'); + + axes(ax_signal); cla; + imagesc(Ysignal(:, :, m), range_ac); hold on; + % set(gca, 'children', flipud(get(gca, 'children'))); + title(sprintf('(Raw-BG) X %d', multi_factor)); + axis equal off tight; + + axes(ax_denoised); cla; + imagesc(Yac(:, :, m), range_ac); + % imagesc(Ybg(:, :, m), [-50, 50]); + title(sprintf('A*C X %d', multi_factor)); + axis equal off tight; + + axes(ax_res); cla; + imagesc(Ysignal(:, :, m)-Yac(:, :, m), range_res); + % set(gca, 'children', flipud(get(gca, 'children'))); + title(sprintf('Residual(Ysignal-Yac) X %d', multi_factor)); + axis equal off tight; + % subplot(4,6, [5,6,11,12]+12); + + drawnow(); + temp = getframe(gcf); + temp = imresize(temp.cdata, [400, 600]); + avi_file.writeVideo(temp); +end + +avi_file.close(); +end + +% bgPermute=Ysignal/1000; +% clear Ysignal +% img1 = max(bgPermute, 0); +% img1 = img1 ./ max(img1(:)); +% outputVideo=VideoWriter([outputdir currentday 'ProcessedSignal.avi']); +% outputVideo.FrameRate=30; +% open(outputVideo); +% +% for p = 1:size(img1,3) +% img = img1(:,:,p); +% writeVideo(outputVideo,img); +% end +% +% close(outputVideo); + + +%Ysignal=reshape(Ysignal,d1,d2,[]); + +% bgPermute=Ysignal/1000; +% clear Ysignal +% img1 = max(bgPermute, 0); +% img1 = img1 ./ max(img1(:)); +% outputVideo=VideoWriter([outputdir currentday 'RawSignal.avi']); +% outputVideo.FrameRate=30; +% open(outputVideo); +% +% for p = 1:size(img1,3) +% img = img1(:,:,p); +% writeVideo(outputVideo,img); +% end +% +% close(outputVideo); diff --git a/BatchVerMO/MoveDiscarded.m b/BatchVerMO/MoveDiscarded.m new file mode 100644 index 0000000..5b8a7ef --- /dev/null +++ b/BatchVerMO/MoveDiscarded.m @@ -0,0 +1,13 @@ +function MoveDiscarded(path2cnmfe,datafolder) +currentFolder = pwd; +cd(datafolder) +mkdir ActuallyUsedInCNMFE +load(path2cnmfe) +Filename=[]; +destination_Filename=[]; +for i=1:length(neuron_batchMO) + filename=neuron_batchMO(i).FileOrigin.name; + destination_filename=['ActuallyUsedInCNMFE' '/' neuron_batchMO(i).FileOrigin.name]; + movefile(filename,destination_filename) +end +cd(currentFolder) \ No newline at end of file diff --git a/BatchVerMO/Over_Days_ResequenceA.m b/BatchVerMO/Over_Days_ResequenceA.m new file mode 100644 index 0000000..99cdc76 --- /dev/null +++ b/BatchVerMO/Over_Days_ResequenceA.m @@ -0,0 +1,36 @@ +function As=Over_Days_ResequenceA(As,correlation_thresh,max2max2nd,skewnessthresh) +% General description: This function puts neurons that are roughly checked(findn1n2) +% as similar over two consecutive days in similar order. This should help +% make cnmfe-BatchVer more robust since the sequence of initiation might +% effect the result of extracted temporal traces. This function intends to +% preserve the sequence of some similar neurons. The idea is illustrated in +% the following example. If in day1 neuron 2 and 3 is tracked similared to +% neuron 4 and 3 in day2. Then the neurons in day2 will be resequenced as 3 +% and 4 while other neurons that have nothing to do with day1's neuron do +% not move. +% Input: +% As={A1,A2,A3,A4...} +% Filters for saying they are the same neuron: correlation_thresh, ratio of first max/second max correlation coef, skewnessthresh. +% Type help findn1n2 for more information for skewnessthresh. +% Output: +% As: re-sequenced As. +% Shijie Gu, techel@live.cn, ShanghaiTech University; Fee Lab at MIT-BCS. + +if or(isempty(As),length(As)==1) + error('As is empty or has only one A in it. No sense to use this function. Please check data selected.') +else + for i=1:length(As)-1 + ns_storage=findn1n2(As{i},As{i+1},correlation_thresh,max2max2nd,skewnessthresh); + if or(isempty(ns_storage),size(ns_storage,1)==1) % no need to re-sequence them. + continue + else + [~,ns_storage_descend_i]=sort(ns_storage(:,1),1,'descend'); + ns_storage=ns_storage(ns_storage_descend_i,:); + new_location=ns_storage(:,2); + all_location=1:size(As{i+1},2); + new_location_old=sort(new_location); + all_location(new_location_old)=new_location; + As{i+1}=As{i+1}(:,all_location); + end + end +end \ No newline at end of file diff --git a/BatchVerMO/Over_Days_ResequenceAForMo.m b/BatchVerMO/Over_Days_ResequenceAForMo.m new file mode 100644 index 0000000..4a742ec --- /dev/null +++ b/BatchVerMO/Over_Days_ResequenceAForMo.m @@ -0,0 +1,44 @@ +function As=Over_Days_ResequenceAForMo(As,correlation_thresh,max2max2nd,skewnessthresh) +% General description: This function is based on Over_Days_ResequenceA. For +% BatchVerMo, it applies the first day's As resequenced result to all the +% other days. Below is the documentation for "Over_Days_ResequenceA". +% This function puts neurons that are roughly checked(findn1n2) +% as similar over two consecutive days in similar order. This should help +% make cnmfe-BatchVer more robust since the sequence of initiation might +% effect the result of extracted temporal traces. This function intends to +% preserve the sequence of some similar neurons. The idea is illustrated in +% the following example. If in day1 neuron 2 and 3 is tracked similared to +% neuron 4 and 3 in day2. Then the neurons in day2 will be resequenced as 3 +% and 4 while other neurons that have nothing to do with day1's neuron do +% not move. +% Input: +% As={A1,A2,A3,A4...} +% Filters for saying they are the same neuron: correlation_thresh, ratio of first max/second max correlation coef, skewnessthresh. +% Type help findn1n2 for more information for skewnessthresh. +% Output: +% As: re-sequenced As. +% Shijie Gu, techel@live.cn, ShanghaiTech University; Fee Lab at MIT-BCS. + +As_all=As; +As=As{1}; +if or(isempty(As),length(As)==1) + error('As is empty or has only one A in it. No sense to use this function. Please check data selected.') +else + for i=1:length(As)-1 + ns_storage=findn1n2(As{i},As{i+1},correlation_thresh,max2max2nd,skewnessthresh); + if or(isempty(ns_storage),size(ns_storage,1)==1) % no need to re-sequence them. + continue + else + [~,ns_storage_descend_i]=sort(ns_storage(:,1),1,'descend'); + ns_storage=ns_storage(ns_storage_descend_i,:); + new_location=ns_storage(:,2); + all_location=1:size(As{i+1},2); + new_location_old=sort(new_location); + all_location(new_location_old)=new_location; + for j=1:length(As_all) + As_all{j}{i+1}=As_all{j}{i+1}(:,all_location); + end + end + end +end +As=As_all; \ No newline at end of file diff --git a/BatchVerMO/Over_Days_findAnn.m b/BatchVerMO/Over_Days_findAnn.m new file mode 100644 index 0000000..c74f046 --- /dev/null +++ b/BatchVerMO/Over_Days_findAnn.m @@ -0,0 +1,28 @@ +function [ns_storage]=Over_Days_findAnn(As,correlation_thresh,max2max2nd,skewnessthresh) +% Input: +% As={A1,A2,A3,A4...} +% Filters for saying they are the same neuron: correlation_thresh, ratio of first max/second max correlation coef, skewnessthresh. +% Type help findn1n2 for more information for skewnessthresh. +% Output: +% ns_storage: each column for each day, index for k in A or C such that +% when you specify A(ns_storage(:,1)) and A(ns_storage(:,2)), +% each row of the permutated A?s from time point 1 and point 2 are the index for the same neuron in the original As + +% Shijie Gu, techel@live.cn (ShanghaiTech University, Harvard-MIT) +if or(isempty(As),length(As)==1) + error('As is empty or has only one A in it. No sense to use this function. Please check data selected.') +elseif length(As)==2; + ns_storage=findn1n2(As{1},As{2},correlation_thresh,max2max2nd,skewnessthresh); +else + ns_storage=findn1n2(As{1},As{2},correlation_thresh,max2max2nd,skewnessthresh); + for i=2:length(As)-1 + ns_next=findn1n2(As{i},As{i+1},correlation_thresh,max2max2nd,skewnessthresh); + [ind_row_storage,ind_row_next] = ismember(ns_storage(:,end),ns_next(:,1)); + ns_storage=ns_storage(ind_row_storage,:); + ind_row_next=ind_row_next(ind_row_next~=0); + ns_next=ns_next(:,2); + ns_next=ns_next(ind_row_next); + ns_storage=[ns_storage ns_next]; + end +end +end \ No newline at end of file diff --git a/BatchVerMO/PartTraces.m b/BatchVerMO/PartTraces.m new file mode 100644 index 0000000..e97b81c --- /dev/null +++ b/BatchVerMO/PartTraces.m @@ -0,0 +1,85 @@ +function [PartC, AllC,kernel_pars, tmp_sn, boundary,neuron_batch,boundary_raw]=PartTraces(neuron_batch) +% This function uses one neurons's calcium traces the temporal traces from all files to estimate +% deconvolution parameters and apply it to all small traces extracted from seperate +% files. It concatenates files by noise-signal-noise so that the +% concatenated signals are coreherent. + +% Outputs: +% PartC: all the neurons' temporal traces concatenated, +% neuron_batch.signal. neurons are put in rows. +% AllC: all the neurons' temporal traces concatenated. +% Similar to PartC, but no deconvolved, only rescaled(normalized by noise) +% Neurons are put in rows. +% kernel_pars: cell array, length of neuron number. Each cell has the +% parameters, 1*N parameter, based on your choice of deconvolution. +% tmp_sn: cell array, length of neuron number. Each cell is a cell array +% of length of file number. Cell{i}{j} is the noise level of neuron +% {i} in file {j} +% PartC_raw(omited, but can always get): +% Cell array, of length of neuron number. Each cell contains the noise-signal-noise +% concatenated temporal traces. +% boundary: cell array, of length of neuron number. Each cell is the time +% boundary for PartC_raw. +% neuron_batch: field 'signal' is filled in. +% boundary_raw: similar to boundary, although it is the boundary for +% PartC. + +% +% Modified by Shijie Gu, ShanghaiTech University; Fee Lab at BCS, MIT. +% Main dependence: deconvolveCa.mat in the original cnmf_e package. + +K=size(neuron_batch(1).rawsignal,1); +PartC=[]; PartC_raw={}; +boundary={}; +boundary_raw=[]; +kernel_pars={}; + +deconv_options_0 = neuron_batch(1).neuron.options.deconv_options; +tmp_sn=cell(1,K); b=cell(1,K); + +for ni=1:K + display(['neuron' num2str(ni)]) + C=[]; bound=[]; bound_raw=[]; + + for fi=1:length(neuron_batch) + C_single=neuron_batch(fi).rawsignal(ni,:); + if range(C)/std(C)>3 + [b{ni}{fi}, tmp_sn{ni}{fi}] = estimate_baseline_noise(C_single); + else + b{ni}{fi} = mean(C_single(C_single + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + + + + + + +
+

CNMF-E (BatchVer)

+

CNMF-E is powerful in extracting neurons from calcium imaging data captured by endoscope, especially in longer data. However, due to the concern of photo-bleaching and other experimental limits and designs, usually our data sets consist of large amount of small(short) data. One can run CNMF-E on all the small data individually yet if you would like to track the same nuerons over days, some extra analysis would be needed. This version helps deal with this situation. Most inspirations of this version derive from ideas implemented in the normal CNMF-E. It has good scalability and manageable memory use. Please read CNMF-E principle before using this version as the documentation assumes the terminology used in normal CNMF-E. See section “Description” for method descirption, and see “Getting Started” to use this version of cnmf-e.

+
+

Description

+

The package uses normal CNMF-E on two different data sets with two different “depth”:

+
    +

  1. full CNMF-E on sample data—“initiation mode”—to get a representative A of this data set;

    2. +

  2. simplified CNMF-E on all data individually, regressing background subtracted data onto A (representative A found in step 1) and C—“massive mode”.

    3. +
+

The following figure shows the general idea of the batched version of CNMF-E.

+
+Figure1, General Description of Batch Version CNMF-E +
+

+

More detailed decription of this version is shown in Figure2 below. You may need to consult the variable summary table below alongside.

+
+Figure2, Detailed Description of Batch Version CNMF-E +
+
+ + +
+
+Table1, Variable Summary Table +
+
+ + +
+
+
+

Prerequisites

+

Very Important 0. Motion corrected data

+

Data must not have motion. Users can tell if there is motion by inspecting the spatial footprint for each sample in the “result files”. Open them as unit8 images in imageJ and paste one image on top of another (select white-zero in “paste control”).

+
    +
  1. Basic cnmf-e
    2. +
+

This package builds on basic cnmf-e. (A summary between the differences between the basic version and this batch version is below.) This not only means you need code from the basic version but also means that certain requirement for data of basic cnmf-e apply here as well, such as motion-corrected data.

+
    +
  1. Cluster-dependancy
    2. +
+

Though this package has fairly adjustable RAM requirement provided by the flexible choice of samples, its normal requirement for 50 GB of RAM makes it cluster-dependent. Except from using small data testing your code, in which case you can set “running_on_cluster=false”, you should normally set “running_on_cluster=true”. Meanwhile, specify workersnum you want to have given your cluster setting in the input section. More will be discussed in section “Getting Started”.

+
    +

  1. Data storage site accessible both on your cluster and your local machine. You will see why in just a minute.

    2. +

  2. Matlab version

    3. +
+

Matlab version may come into play in terms of whether the package will run smooth or not. The package is tested during development on MATLAB2016b for the parts run on cluster. Although, the beginning part of the package is simple input/output specification, which can be run on 2014b or onwards.

+
+
+

Getting Started

+

This package is of two parts: (1) cnmfeBatchVer_LocalPart (2) cnmfeBatchVer_ClusterPart The first part is input/output specification. It runs on your local computer. The purpose of this part is to make sure sample data and experimental data are properly chosen. The variables will be saved and will be used in cnmfeBatchVer_ClusterPart, which is all the actual work of neuron extraction and output parsing.

+
+

(1) cnmfeBatchVer_LocalPart

+
+

A. Important directories and parameters

+
    +

  1. Open the template/demp “cnmfeBatchVer_LocalPart” in the “BatchVer” folder. If not, specify code directory and somthing like addpath(genpath(codeDir));

    2. +

  2. Fill in where the code will be for the cluster.

    3. +

  3. In InputOutput(), fill in arguments where all the directory should be with respect to your local machine/PC, not the cluster. This function involves an interactive process and possibly a pop-out window to choose sample files if you set ‘SamplingMethod’=‘manual’. Alternatively, choose ‘SamplingMethod’=‘auto’. You will only need to input a number when prompted. Type “help InputOutput()” in MATLAB command window for more infomation. Note, data types that are supported are the same as that in the original CNMF-E.

    4. +

  4. After this section, section1.1 helps you to change directory pre-fix from your local machine to that on cluster. Users are encouraged to look into InputOutput() and section 1.1 to change the syntax/method that easily specify directories given the user’s data storage structure.

    5. +

  5. Section2 specifies some parameters used in normal cnmf-e. There are a few more parameters that can tuned but are not listed here. User can modify this according to their needs, but most important ones that are able to make a significant difference are listed here. One different “parameter” is the last variable, “namepattern”. In cnmfe(BatchVer), after extracting each sample data, a plot of A will be saved in the working folder of the cluster. In the current release, the plot will be named with filename(namepattern). In addition, in cnmfe(BatchVer), initiation process of each sample data has a plot showing the min_PNR*min_corr curve and the seeds of neuron. This plot is intended to help user to better find a suitable min_PNR. This plot uses the same naming strategy.

    6. +

  6. Section3 is some parameters specifically important to BatchVer. Type help Over_Days_findAnn for explaination and help MergeAC. There is no general rule; it depends on your data. Please try some to figure out a suitable value for you. Note that the “merge_thr_2” here will be used in MergeAC, the BatchVer specific merging, while “merge_thr” in section2 is used for for normal CNMF-E neuron merging in initiation and iteration process.

    7. +

  7. Section4 specifies workers of matlab parallel pool. This number will depend on the cluster you use and the CPU number you request. Consult your cluster documentation on this. With this workersnum, cnmfeBatchVer_ClusterPart has functions written to create a cluster and a parpool within it. The design caters to cluster use. The functions for these are in a seperate folder “HandlingParforbyGalen” which is written by Galen Lynch.

    8. +

  8. Save all these variables to the outputdir with respect to your local machine, which will be read on cluster from “outputdir”.

    9. +
+
+
+

B. Script for cluster

+

Fill in the template in Part B to make a script for cluster. The example here is for clusters using SLURM for job management. To make the script look clearer, we reproduce results here.

+
#!/bin/bash
+#SBATCH -n 1
+#SBATCH --cpus-per-task=8
+#SBATCH --mem=200000
+#SBATCH -t 0-10:00
+#SBATCH --time-min=0-01:00
+#SBATCH -o /net/feevault/data0/shared/EmilyShijieShared/BatchResult/6938FirstFewDaysForBatch/job_%A.out
+#SBATCH -e /net/feevault/data0/shared/EmilyShijieShared/BatchResult/6938FirstFewDaysForBatch/job_%A.err
+cd /net/feevault/data0/shared/EmilyShijieShared/BatchResult/6938FirstFewDaysForBatch/
+module add mit/matlab/2016b
+

The matlab command is essentially:

+
load(fullfile('/net/feevault/data0/shared/EmilyShijieShared/BatchResultTest/','LogisticscnmfeBatchVer.mat'))
+addpath(genpath(codeDir));
+cnmfeBatchVer_ClusterPart
+

First load variables made by cnmfeBatchVer_LocalPart. Then add code directory. Run cnmfeBatchVer_ClusterPart.

+
+
+
+

(2) cnmfeBatchVer_ClusterPart

+

From now on, users do not need to look into the code. Just let the clsuter run. Basic ideas in this black box are shown in Figure2 above. Users who would like to improve the method can look into key functions in the figure. They all have documentations inside the functions and can be accessed with “help”.

+
+

Output description

+

The output is in CNMFE_BatchVer.mat, in which neuron_batch is the structure of the result. See Key Variable Table. Other outputs are meant for parameter choice or for parameter monitor in case somthing goes wrong. See Parameter choice (PNR) for more of this.

+
+
+
+
+

Summary of differences between CNMF-E (basic) and CNMF-E (BatchVer)

+

Compared to normal CNMF-E with many rounds of iteration of background, spatial and temporal update and merging neurons for each file, BatchVer splits the process on different scales. For sampling, normal rounds of iteration of spatial and temporal update apply. From them, we get a comprehensive and relatively decent A. Then, for all files, use this A to find C on roughly background subtracted data. After one round of temporal update (de-noise and deconvolution), output the result. The output that will be primarily used is A*C. The design of this version sacrifices some precision on the extracted result but enable users to compare activity of neurons over days.

+
+
+

Parameter choice

+

There are many options in CNMF-E that can be used in your data’s favour. Here we provide some guidelines on gSig, gSiz, PNR.

+
+

gSig and gSiz

+
    +

  • Amount gSiz/gSig ratio is critical. Only some range is good. For example, some range may be 1.2-2. Below 1.2, too few neurons selected, above 2, too messy. There is also one ratio, around which no matter what absolute gSiz and gSig is, results look similar in every aspect.

    • +

  • Energy/spatial spread (1)Sometimes neurons extracted are with “ripples of tails”. Some amounts of tails give us dendrite/axon information, yet too much of this is suspicious in that noise contamination is more likely.

    • +
+
    +
  1. Sometimes neurons have “energies” “spread out” rather than focused. This can happen when neurons are out of focus or are on the edge of the FOV. Those neurons often are bigger than most of the central neurons. The common point of (1) (2) is how much spatial footprint spreadout.
    2. +
+

Two factors can affect what level of spread out spatial footprint show up in the final extraction: absolute value of gSiz and gSiz/gSig ratio. Bigger gSiz => more tails. Bigger gSiz/gSig ratio => more tails.

+

In general: gSiz < some level is good. gSiz/gSig ratio <= some level is good.

+

If you have a lot of data to operate on, you may end up choosing one conservative parameter for all of them.

+
+
+

PNR

+

Compared to min_corr, min_PNR seems to have a much bigger effect. For each sample with initiation involved, a plot is drawn to show the min_corr, min_PNR threshold and the initiated seeds. It is carried out by the newly intoduced method drawPNRCn(min_pnr,min_corr). You can change PNR and see how many seeds get involved. The number on the plot next to each seed correspond to the final neuron number(column/row number) in the neuron.A and neuron.C. You will need to couple this figure with the output of viewNeuron for each sample, each in a folder (spatial and temporal trace for each neuron).

+
+
+

About RAM and time choice

+

RAM need is roughly in linear relationship with the amount of sample data you choose. Time choice depends on the amount of both sample data and all data for extraction.

+
+
+
+

Notes

+

This version keeps some features of basic version intact, one of which is the option in initiating in patches. PNR/Correlation diagnose plot is not supported in patch option though interested users are encouraged to modify it. Since we are already operating on small data sets (in sampling stage) and that patching field of view is used to reduce memory use, patching is not necessary here. In addition, there might be some artifacts with the patch option. In short, patch view is not recommended.

+
+
+

Authors

+

Pengcheng Zhou initiated the idea. Emily Mackevicius and Shijie Gu developed the idea. Shijie Gu implemented this version.

+
    +
  • Shijie Gu email
    • +
  • Emily Mackevicius email
    • +
  • Pengcheng Zhou
    • +
+
+
+

Acknowledgments

+

Galen Lynch provides two functions that are necessary for smooth parfor function on cluster. * Galen Lynch email

+
+
+ + + + +
+ + + + + + + + diff --git a/BatchVerMO/ReadMe/ReadMeHTML/README-BatchVer.md b/BatchVerMO/ReadMe/ReadMeHTML/README-BatchVer.md new file mode 100644 index 0000000..1aa40a1 --- /dev/null +++ b/BatchVerMO/ReadMe/ReadMeHTML/README-BatchVer.md @@ -0,0 +1,149 @@ +# CNMF-E (BatchVer) + +CNMF-E is powerful in extracting neurons from calcium imaging data captured by endoscope, especially in longer data. However, due to the concern of photo-bleaching and other experimental limits and designs, usually our data sets consist of large amount of small(short) data. One can run CNMF-E on all the small data individually yet if you would like to track the same nuerons over days, some extra analysis would be needed. This version helps deal with this situation. Most inspirations of this version derive from ideas implemented in the normal CNMF-E. It has good scalability and manageable memory use. Please read CNMF-E principle before using this version as the documentation assumes the terminology used in normal CNMF-E. See section "Description" for method descirption, and see "Getting Started" to use this version of cnmf-e. + +## Description +The package uses normal CNMF-E on two different data sets with two different "depth": + + (1) full CNMF-E on sample data---"initiation mode"---to get a representative A of this data set; + + (2) simplified CNMF-E on all data individually, regressing background subtracted data onto A (representative A found in step 1) and C---"massive mode". + + The following figure shows the general idea of the batched version of CNMF-E. + +
Figure1, General Description of Batch Version CNMF-E
+![](/Users/gushijie/Documents/MATLAB/CaImaging/cnmf-e/BatchVer/ReadMe/Overview.png) + + More detailed decription of this version is shown in Figure2 below. You may need to consult the variable summary table below alongside. + +
Figure2, Detailed Description of Batch Version CNMF-E
+![](/Users/gushijie/Documents/MATLAB/CaImaging/cnmf-e/BatchVer/ReadMe/July10th.png) + + +
Table1, Variable Summary Table
+![](/Users/gushijie/Documents/MATLAB/CaImaging/cnmf-e/BatchVer/ReadMe/VariablesJuly10th.png) + + +## Prerequisites +__*Very Important*__ +0. Motion corrected data + +Data must not have motion. Users can tell if there is motion by inspecting the spatial footprint for each sample in the "result files". Open them as unit8 images in imageJ and paste one image on top of another (select white-zero in "paste control"). + +1. Basic cnmf-e + +This package builds on basic cnmf-e. (A summary between the differences between the basic version and this batch version is below.) This not only means you need code from the basic version but also means that certain requirement for data of basic cnmf-e apply here as well, such as motion-corrected data. + +2. Cluster-dependancy + +Though this package has fairly adjustable RAM requirement provided by the flexible choice of samples, its normal requirement for 50 GB of RAM makes it cluster-dependent. Except from using small data testing your code, in which case you can set "running_on_cluster=false", you should normally set "running_on_cluster=true". Meanwhile, specify workersnum you want to have given your cluster setting in the input section. More will be discussed in section "Getting Started". + +3. Data storage site accessible both on your cluster and your local machine. +You will see why in just a minute. + +4. Matlab version + +Matlab version may come into play in terms of whether the package will run smooth or not. The package is tested during development on MATLAB2016b for the parts run on cluster. Although, the beginning part of the package is simple input/output specification, which can be run on 2014b or onwards. + +## Getting Started + +This package is of two parts: (1) cnmfeBatchVer_LocalPart (2) cnmfeBatchVer_ClusterPart +The first part is input/output specification. It runs on your local computer. The purpose of this part is to make sure sample data and experimental data are properly chosen. The variables will be saved and will be used in cnmfeBatchVer_ClusterPart, which is all the actual work of neuron extraction and output parsing. + +### (1) cnmfeBatchVer_LocalPart +#### A. Important directories and parameters +1. Open the template/demp "cnmfeBatchVer_LocalPart" __in the "BatchVer" folder__. If not, specify code directory and somthing like addpath(genpath(codeDir)); + +2. Fill in where the code will be __for the cluster__. + +3. In InputOutput(), fill in arguments where all the directory should be with respect to __your local machine/PC__, not the cluster. This function involves an interactive process and possibly a pop-out window to choose sample files if you set 'SamplingMethod'='manual'. Alternatively, choose 'SamplingMethod'='auto'. You will only need to input a number when prompted. Type "help InputOutput()" in MATLAB command window for more infomation. Note, data types that are supported are the same as that in the original CNMF-E. + +4. After this section, section1.1 helps you to change directory pre-fix from your local machine to that on cluster. Users are encouraged to look into InputOutput() and section 1.1 to change the syntax/method that easily specify directories given the user's data storage structure. + +5. Section2 specifies some parameters used in normal cnmf-e. There are a few more parameters that can tuned but are not listed here. User can modify this according to their needs, but most important ones that are able to make a significant difference are listed here. One different "parameter" is the last variable, "namepattern". In cnmfe(BatchVer), after extracting each sample data, a plot of A will be saved in the working folder of the cluster. In the current release, the plot will be named with filename(namepattern). In addition, in cnmfe(BatchVer), initiation process of each sample data has a plot showing the min_PNR*min_corr curve and the seeds of neuron. This plot is intended to help user to better find a suitable min_PNR. This plot uses the same naming strategy. + +6. Section3 is some parameters specifically important to BatchVer. Type help Over_Days_findAnn for explaination and help MergeAC. There is no general rule; it depends on your data. Please try some to figure out a suitable value for you. Note that the "merge_thr_2" here will be used in MergeAC, the BatchVer specific merging, while "merge_thr" in section2 is used for for normal CNMF-E neuron merging in initiation and iteration process. + +7. Section4 specifies workers of matlab parallel pool. This number will depend on the cluster you use and the CPU number you request. Consult your cluster documentation on this. With this workersnum, cnmfeBatchVer_ClusterPart has functions written to create a cluster and a parpool within it. The design caters to cluster use. The functions for these are in a seperate folder "HandlingParforbyGalen" which is written by Galen Lynch. + +8. Save all these variables to the outputdir with respect to your local machine, which will be read on cluster from "outputdir". + +#### B. Script for cluster + +Fill in the template in Part B to make a script for cluster. The example here is for clusters using SLURM for job management. +To make the script look clearer, we reproduce results here. +``` +#!/bin/bash +#SBATCH -n 1 +#SBATCH --cpus-per-task=8 +#SBATCH --mem=200000 +#SBATCH -t 0-10:00 +#SBATCH --time-min=0-01:00 +#SBATCH -o /net/feevault/data0/shared/EmilyShijieShared/BatchResult/6938FirstFewDaysForBatch/job_%A.out +#SBATCH -e /net/feevault/data0/shared/EmilyShijieShared/BatchResult/6938FirstFewDaysForBatch/job_%A.err +cd /net/feevault/data0/shared/EmilyShijieShared/BatchResult/6938FirstFewDaysForBatch/ +module add mit/matlab/2016b +``` +The matlab command is essentially: +``` +load(fullfile('/net/feevault/data0/shared/EmilyShijieShared/BatchResultTest/','LogisticscnmfeBatchVer.mat')) +addpath(genpath(codeDir)); +cnmfeBatchVer_ClusterPart +``` +First load variables made by cnmfeBatchVer_LocalPart. Then add code directory. Run cnmfeBatchVer_ClusterPart. + +### (2) cnmfeBatchVer_ClusterPart + +From now on, users do not need to look into the code. Just let the clsuter run. Basic ideas in this black box are shown in Figure2 above. Users who would like to improve the method can look into key functions in the figure. They all have documentations inside the functions and can be accessed with "help". + +#### Output description +The output is in CNMFE_BatchVer.mat, in which neuron_batch is the structure of the result. See Key Variable Table. Other outputs are meant for parameter choice or for parameter monitor in case somthing goes wrong. See Parameter choice (PNR) for more of this. + +## Summary of differences between CNMF-E (basic) and CNMF-E (BatchVer) + +Compared to normal CNMF-E with many rounds of iteration of background, spatial and temporal update and merging neurons for each file, BatchVer splits the process on different scales. For sampling, normal rounds of iteration of spatial and temporal update apply. From them, we get a comprehensive and relatively decent A. Then, for all files, use this A to find C on roughly background subtracted data. After one round of temporal update (de-noise and deconvolution), output the result. The output that will be primarily used is A*C. The design of this version sacrifices some precision on the extracted result but enable users to compare activity of neurons over days. + +## Parameter choice + +There are many options in CNMF-E that can be used in your data's favour. Here we provide some guidelines on gSig, gSiz, PNR. + +### gSig and gSiz +* **Amount** +gSiz/gSig ratio is critical. Only some range is good. For example, some range may be 1.2-2. Below 1.2, too few neurons selected, above 2, too messy. There is also one ratio, around which no matter what absolute gSiz and gSig is, results look similar in every aspect. + +* **Energy/spatial spread** +(1)Sometimes neurons extracted are with "ripples of tails". Some amounts of tails give us dendrite/axon information, yet too much of this is suspicious in that noise contamination is more likely. + +(2) Sometimes neurons have "energies" "spread out" rather than focused. This can happen when neurons are out of focus or are on the edge of the FOV. Those neurons often are bigger than most of the central neurons. +The common point of (1) (2) is how much spatial footprint spreadout. + +Two factors can affect what level of spread out spatial footprint show up in the final extraction: absolute value of gSiz and gSiz/gSig ratio. +Bigger gSiz => more tails. +Bigger gSiz/gSig ratio => more tails. + +In general: gSiz < some level is good. +gSiz/gSig ratio <= some level is good. + +If you have a lot of data to operate on, you may end up choosing one conservative parameter for all of them. + +### PNR +Compared to min_corr, min_PNR seems to have a much bigger effect. For each sample with initiation involved, a plot is drawn to show the min_corr, min_PNR threshold and the initiated seeds. It is carried out by the newly intoduced method drawPNRCn(min_pnr,min_corr). You can change PNR and see how many seeds get involved. The number on the plot next to each seed correspond to the final neuron number(column/row number) in the neuron.A and neuron.C. You will need to couple this figure with the output of viewNeuron for each sample, each in a folder (spatial and temporal trace for each neuron). + +### About RAM and time choice +RAM need is roughly in linear relationship with the amount of sample data you choose. Time choice depends on the amount of both sample data and all data for extraction. + +## Notes + +This version keeps some features of basic version intact, one of which is the option in initiating in patches. PNR/Correlation diagnose plot is not supported in patch option though interested users are encouraged to modify it. Since we are already operating on small data sets (in sampling stage) and that patching field of view is used to reduce memory use, patching is not necessary here. In addition, there might be some artifacts with the patch option. In short, patch view is not recommended. + +## Authors +Pengcheng Zhou initiated the idea. Emily Mackevicius and Shijie Gu developed the idea. Shijie Gu implemented this version. + +* **Shijie Gu** [email](techel@live.cn) +* **Emily Mackevicius** [email](elm@mit.edu) +* **Pengcheng Zhou** + +## Acknowledgments +Galen Lynch provides two functions that are necessary for smooth parfor function on cluster. +* **Galen Lynch** [email](glynch@mit.edu) + diff --git a/BatchVerMO/ReadMe/ReadMeHTML/README-Template.html b/BatchVerMO/ReadMe/ReadMeHTML/README-Template.html new file mode 100644 index 0000000..3aca0a8 --- /dev/null +++ b/BatchVerMO/ReadMe/ReadMeHTML/README-Template.html @@ -0,0 +1,237 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + + + + + + +
+

CNMF-E (BatchVer)

+

CNMF-E is powerful in extracting neurons from calcium imaging data captured by endoscope, especially in longer data. However, due to the concern of photo-bleaching and other experimental limits and designs, usually our data sets consist of large amount of small(short) data. One can run CNMF-E on all the small data individually yet if you would like to track the same nuerons over days, some extra analysis would be needed. This version helps deal with this situation. Most inspirations of this version derive from ideas implemented in the normal CNMF-E. It has good scalability and manageable memory use. Please read CNMF-E principle before using this version as the documentation assumes the terminology used in normal CNMF-E. See section “Description” for method descirption, and see “Getting Started” to use this version of cnmf-e.

+
+

Descirption

+

The package uses normal CNMF-E on two different data sets with two different “depth”:

+
    +

  1. full CNMF-E on sample data—“initiation mode”—to get a representative A of this data set;

    2. +

  2. simplified CNMF-E on all data individually, regressing background subtracted data onto A (representative A found in step 1) and C—“massive mode”.

    3. +
+

The following figure shows the general idea of the batched version of CNMF-E.

+
+Figure1, General Description of Batch Version CNMF-E +

Figure1, General Description of Batch Version CNMF-E

+
+

More detailed decription of this version is shown in Figure2 below. Figure2, General Description of Batch Version CNMF-E

+
+

Prerequisites

+

What things you need to install the software and how to install them

+
Give examples
+
+
+

Installing

+

A step by step series of examples that tell you have to get a development env running

+

Say what the step will be

+
Give the example
+

And repeat

+
until finished
+

End with an example of getting some data out of the system or using it for a little demo

+
+
+
+

Running the tests

+

Explain how to run the automated tests for this system

+
+

Break down into end to end tests

+

Explain what these tests test and why

+
Give an example
+
+
+

And coding style tests

+

Explain what these tests test and why

+
Give an example
+
+
+
+

Deployment

+

Add additional notes about how to deploy this on a live system

+
+
+

Built With

+
    +
  • Dropwizard - The web framework used
    • +
  • Maven - Dependency Management
    • +
  • ROME - Used to generate RSS Feeds
    • +
+
+
+

Contributing

+

Please read CONTRIBUTING.md for details on our code of conduct, and the process for submitting pull requests to us.

+
+
+

Versioning

+

We use SemVer for versioning. For the versions available, see the tags on this repository.

+
+
+

Authors

+ +

See also the list of contributors who participated in this project.

+
+
+

License

+

This project is licensed under the MIT License - see the LICENSE.md file for details

+
+
+

Acknowledgments

+
    +
  • Hat tip to anyone who’s code was used
    • +
  • Inspiration
    • +
  • etc
    • +
+
+
+ + + + +
+ + + + + + + + diff --git a/BatchVerMO/ReadMe/ReadMeHTML/README-Template.md b/BatchVerMO/ReadMe/ReadMeHTML/README-Template.md new file mode 100755 index 0000000..996178a --- /dev/null +++ b/BatchVerMO/ReadMe/ReadMeHTML/README-Template.md @@ -0,0 +1,98 @@ +# CNMF-E (BatchVer) + +CNMF-E is powerful in extracting neurons from calcium imaging data captured by endoscope, especially in longer data. However, due to the concern of photo-bleaching and other experimental limits and designs, usually our data sets consist of large amount of small(short) data. One can run CNMF-E on all the small data individually yet if you would like to track the same nuerons over days, some extra analysis would be needed. This version helps deal with this situation. Most inspirations of this version derive from ideas implemented in the normal CNMF-E. It has good scalability and manageable memory use. Please read CNMF-E principle before using this version as the documentation assumes the terminology used in normal CNMF-E. See section "Description" for method descirption, and see "Getting Started" to use this version of cnmf-e. + +## Descirption +The package uses normal CNMF-E on two different data sets with two different "depth": + + (1) full CNMF-E on sample data---"initiation mode"---to get a representative A of this data set; + + (2) simplified CNMF-E on all data individually, regressing background subtracted data onto A (representative A found in step 1) and C---"massive mode". + + The following figure shows the general idea of the batched version of CNMF-E. + +![Figure1, General Description of Batch Version CNMF-E](/Users/gushijie/Documents/MATLAB/CaImaging/cnmf-e/BatchVer/ReadMe/General.png) + + More detailed decription of this version is shown in Figure2 below. +![Figure2, General Description of Batch Version CNMF-E](/Users/gushijie/Documents/MATLAB/CaImaging/cnmf-e/BatchVer/ReadMe/Detail.png) + +### Prerequisites + +What things you need to install the software and how to install them + +``` +Give examples +``` + +### Installing + +A step by step series of examples that tell you have to get a development env running + +Say what the step will be + +``` +Give the example +``` + +And repeat + +``` +until finished +``` + +End with an example of getting some data out of the system or using it for a little demo + +## Running the tests + +Explain how to run the automated tests for this system + +### Break down into end to end tests + +Explain what these tests test and why + +``` +Give an example +``` + +### And coding style tests + +Explain what these tests test and why + +``` +Give an example +``` + +## Deployment + +Add additional notes about how to deploy this on a live system + +## Built With + +* [Dropwizard](http://www.dropwizard.io/1.0.2/docs/) - The web framework used +* [Maven](https://maven.apache.org/) - Dependency Management +* [ROME](https://rometools.github.io/rome/) - Used to generate RSS Feeds + +## Contributing + +Please read [CONTRIBUTING.md](https://gist.github.com/PurpleBooth/b24679402957c63ec426) for details on our code of conduct, and the process for submitting pull requests to us. + +## Versioning + +We use [SemVer](http://semver.org/) for versioning. For the versions available, see the [tags on this repository](https://github.com/your/project/tags). + +## Authors + +* **Billie Thompson** - *Initial work* - [PurpleBooth](https://github.com/PurpleBooth) + +See also the list of [contributors](https://github.com/your/project/contributors) who participated in this project. + +## License + +This project is licensed under the MIT License - see the [LICENSE.md](LICENSE.md) file for details + +## Acknowledgments + +* Hat tip to anyone who's code was used +* Inspiration +* etc + diff --git a/BatchVerMO/ReadMe/Variables.docx b/BatchVerMO/ReadMe/Variables.docx new file mode 100644 index 0000000..bcb3552 Binary files /dev/null and b/BatchVerMO/ReadMe/Variables.docx differ diff --git a/BatchVerMO/ReadMe/Variables.pdf b/BatchVerMO/ReadMe/Variables.pdf new file mode 100644 index 0000000..a25e653 Binary files /dev/null and b/BatchVerMO/ReadMe/Variables.pdf differ diff --git a/BatchVerMO/ReadMe/VariablesJuly10th.pdf b/BatchVerMO/ReadMe/VariablesJuly10th.pdf new file mode 100644 index 0000000..2fc5cff Binary files /dev/null and b/BatchVerMO/ReadMe/VariablesJuly10th.pdf differ diff --git a/BatchVerMO/ReadMe/variables1.pdf b/BatchVerMO/ReadMe/variables1.pdf new file mode 100644 index 0000000..7141d62 Binary files /dev/null and b/BatchVerMO/ReadMe/variables1.pdf differ diff --git a/BatchVerMO/ReducingA.m b/BatchVerMO/ReducingA.m new file mode 100644 index 0000000..289b5c1 --- /dev/null +++ b/BatchVerMO/ReducingA.m @@ -0,0 +1,22 @@ +function weightedA=ReducingA(Aunique,STDunique) +%Description: Reduce A for those that cannot be merged but has different values in each +% sampled file. General method is use std as weight to merge +% these. +%Input: two cells each contains Aunique,STDunique of each file. +%Output:One big weightedA. +%author: Shijie Gu, techel@live.cn, (ShanghaiTech University; Fee Lab at MIT-BCS) + A=cat(2,Aunique{:}); + STDsum=sum(cat(3,STDunique{:}),3); + STDcat=cat(2,STDunique{:}); + STDsumcat=repmat(STDsum,1,length(STDunique)); + aveSTD=STDcat./STDsumcat; + %aveSTD=cellfun(@rdivide,STDunique,STDsum,'UniformOutput', false); + catSTD=diag(aveSTD); + + weightedA=A*catSTD; + d=size(A,1); + K=size(Aunique{1},2); + + weightedA=reshape(weightedA,d,K,[]); + weightedA=sum(weightedA,3); +end diff --git a/BatchVerMO/ShowMerge.m b/BatchVerMO/ShowMerge.m new file mode 100644 index 0000000..2ac33a7 --- /dev/null +++ b/BatchVerMO/ShowMerge.m @@ -0,0 +1,11 @@ +picDir='/Users/gushijie/Documents/Fee/BatchedVerDebug/6922/'; +picdir=fullfile(picDir,'*CaELM*'); +piclist=dir(picdir); +C=imread(fullfile(picDir,piclist(1).name)); +for i=2:numel(piclist) + pic=imread(fullfile(picDir,piclist(i).name)); + C= imfuse(C,pic,'blend'); +end + + + diff --git a/BatchVerMO/Show_PartTrace.m b/BatchVerMO/Show_PartTrace.m new file mode 100644 index 0000000..c3dfd54 --- /dev/null +++ b/BatchVerMO/Show_PartTrace.m @@ -0,0 +1,15 @@ +neuronnum=3; +figure +ax(1)=subplot(2,1,1); +plot(PartC_raw{neuronnum}) +hold on +plot(boundary_raw{neuronnum}'*[1 1], [0 max(PartC_raw{neuronnum})], ':', 'color', .7*[1 1 1]); +% plot(AllC(neuronnum,:)) +% hold on +% plot(boundary_raw{neuronnum}'*[1 1], [0 max(AllC(neuronnum,:))], ':', 'color', .7*[1 1 1]); + +ax(2)=subplot(2,1,2); +plot(PartC{neuronnum}) +hold on +plot(boundary_raw{neuronnum}'*[1 1], [0 max(PartC{neuronnum})], ':', 'color', .7*[1 1 1]); +linkaxes(ax) \ No newline at end of file diff --git a/BatchVerMO/SomeAutomatingProcess.asv b/BatchVerMO/SomeAutomatingProcess.asv new file mode 100644 index 0000000..2d9fb36 --- /dev/null +++ b/BatchVerMO/SomeAutomatingProcess.asv @@ -0,0 +1,29 @@ +corr=results.THRESH.Corr; +pnr=results.THRESH.PNR; +CorrOut=results.THRESH.CorrOut; +PNROut=results.THRESH.PNROut; +plot(corr,'*b') +hold on +plot(CorrOut,'*r') +%% +HY00=sort(HY0,2,'descend'); +%MX = mean(HY00(:,floor(size(HY0,2).*0.02)),2); +MX = mean(HY00(:,floor(size(HY0,2).*0.02)),2); +MEAN=mean(HY0,2); +STD=std(HY0,1,2); +deviation_ind=abs((MX-MEAN))./STD>4; +%deviation_sub=ind2sub([300,400],deviation_ind); +Corr_dev=Cn(deviation_ind); +PNR_dev=PNR(deviation_ind); +%% +nonzero=CorrOut>0; +plot(CorrOut(nonzero),PNROut(nonzero),'*r') +matr=[corr CorrOut(nonzero); pnr PNROut(nonzero)]'; +[idx,C]=kmeans(matr,4); +plot(C(:,1),C(:,2),'*k','MarkerSize',15) +plot(matr(idx==1,1),matr(idx==1,2),'r.','MarkerSize',12) +hold on +plot(matr(idx==2,1),matr(idx==2,2),'b.','MarkerSize',12) +plot(matr(idx==3,1),matr(idx==3,2),'g.','MarkerSize',12) +plot(Corr_dev,PNR_dev,'*k','MarkerSize',8) +plot(matr(idx==4,1),matr(idx==4,2),'y.','MarkerSize',12) \ No newline at end of file diff --git a/BatchVerMO/SomeAutomatingProcess.m b/BatchVerMO/SomeAutomatingProcess.m new file mode 100644 index 0000000..130ae4b --- /dev/null +++ b/BatchVerMO/SomeAutomatingProcess.m @@ -0,0 +1,67 @@ +load('X:\elm\ProcessedCalciumData\AllRows\7030\ActuallyUsedInCNMFE\CaELM_row4444_b7030_20170612_145529.mat') +load('X:\shared\EmilyShijieShared\CaExtraction\7030\LogisticscnmfeBatchVer612.mat') +%% +neuron_full.options.d1=300; +neuron_full.options.d2=400; +[results, center, Cn, PNR, save_avi] = greedyROI_endoscope(double(Y), [], neuron_full.options,false,false); +%% +HY=results.HY; +pars=[]; +localmax=results.ind_localmax; +for i=1:numel(results.ind_localmax) + y=HY(localmax(i),:); + sn = GetSn(y); + % estimate time constant + switch neuron_full.options.deconv_options.type + case 'ar1' + par = estimate_time_constant(y, 1, sn); + if length(par)~=1 + par=0;end + pars(i)=par; + end +end +%% +PNR0=results.PNR0; PNR_maxes=PNR0(localmax); PNR_ind=PNR_maxes>results.min_pnr; +test=[pars;PNR_ind']; +%% +corr=results.THRESH.Corr; +pnr=results.THRESH.PNR; +CorrOut=results.THRESH.CorrOut; +PNROut=results.THRESH.PNROut; +%% +PNR_all=reshape(PNR,300*400,1); +Cn=reshape(Cn,300*400,1); +%% +[coeff2,score2,latent2,tsquared2,explained2,mu2] = pca([PNR_all, Cn],'Centered',false); +t = score2(:,1)*coeff2(:,1)'; +%% +figure +subplot(1,2,1) +scatter(score(:,1),score(:,2)) +subplot(1,2,2) +scatter(Cn,PNR_all) +%% +t = score(:,1)*coeff1(:,1)' + repmat(mu,size(pixels,1),1); +%% +HY00=sort(HY0,2,'descend'); +%MX = mean(HY00(:,floor(size(HY0,2).*0.02)),2); +MX = mean(HY00(:,8),2); +MEAN=mean(HY0,2); +STD=std(HY0,1,2); +deviation_ind=abs((MX-MEAN))./STD>4; +%deviation_sub=ind2sub([300,400],deviation_ind); +Corr_dev=Cn(deviation_ind); +PNR_dev=PNR(deviation_ind); +%% +nonzero=CorrOut>0; +plot(CorrOut(nonzero),PNROut(nonzero),'*r') +matr=[corr CorrOut(nonzero); pnr PNROut(nonzero)]'; +[idx,C]=kmeans(matr,5); +plot(C(:,1),C(:,2),'*k','MarkerSize',15) +plot(matr(idx==1,1),matr(idx==1,2),'r.','MarkerSize',12) +hold on +plot(matr(idx==2,1),matr(idx==2,2),'b.','MarkerSize',12) +plot(matr(idx==3,1),matr(idx==3,2),'g.','MarkerSize',12) +plot(Corr_dev,PNR_dev,'*k','MarkerSize',8) +plot(matr(idx==4,1),matr(idx==4,2),'y.','MarkerSize',12) +plot(matr(idx==5,1),matr(idx==5,2),'y.','MarkerSize',12) \ No newline at end of file diff --git a/BatchVerMO/ViewTraces.m b/BatchVerMO/ViewTraces.m new file mode 100644 index 0000000..3cdbd24 --- /dev/null +++ b/BatchVerMO/ViewTraces.m @@ -0,0 +1,104 @@ +% K=size(neuron_batch(1).neuron.A,2); +% AllC=AllTraces(neuron_batch); +% for i=1:K +% subplot(K,1,i) +% hold on +% plot(AllC(i,:)) +% end + +% newIDs=sampleresult.newIDs; +% A0s=sampleresult.A0s; +% [~,size2]=cellfun(@size,A0s); +% sizecumsum=cumsum(size2); + +%C=cat(2,ACS.Cin); +neuronums=[1,60]; +K=length(neuronums); +for i=1:K + I=neuronums(i); + subplot(K,1,i) + hold on + plot(C(I,:)) + title(['original neuron ' num2str(I)]) +end +%% +%A=cat(2,A0s{:}); +[AllC, ~]=AllTracesfromACS(ACS); +AllC=AllC(:,:); + +[AllC_raw, ~]=AllTracesfromACS(ACS,true); +AllC_raw=AllC_raw(:,:); + +%newIDs=Day4_newIDs; +newIDs_nums=[1]; +neuronums=[]; +for i=1:length(newIDs_nums) + neuronums=[neuronums; newIDs{newIDs_nums(i)}]; +end +%% +neuronums=[1 57 19 60]; +K=length(neuronums); +figure +for i=1:K + I=neuronums(i); + subplot(K,1,i) + plot(AllC_raw(I,:),'r') + hold on + plot(AllC(I,:),'b') + title(['original neuron ' num2str(I)]) + +end + + +%% +K=length(neuronums); +for i=1:K + I=neuronums(i); + subplot(K,1,i) + hold on + Areshaped=reshape(A(:,I),300,400); + imagesc(Areshaped) + title(['original neuron ' num2str(I)]) +end +%% +newIDs_nums=[20,42]; +newIDs=DayAll_newIDs; +cum_sum=cumsum(cellfun(@(x) size(x,2),M1{1})); +file_num=[]; +actual_num=[]; +neuronums=[]; +for i=1:length(newIDs_nums) + neuronums=[neuronums; newIDs{newIDs_nums(i)}]; + for j=1:length(newIDs{newIDs_nums(i)}) + x_=newIDs{newIDs_nums(i)}; + x=x_(j); + k=find((cum_sum>=x),1); + file_num=[file_num k]; + if k>1 + x_actual=x-cum_sum(k-1); + else + x_actual=x; + end + actual_num=[actual_num x_actual]; + end +end +%% + +K=length(neuronums); +for i=1:K + I=neuronums(i); + subplot(K,1,i) + hold on + plot(AllC(I,:)) + title(['original neuron ' num2str(I)]) +end + +%% Just plotting from ACS +neurons=[20,21]; +K=length(neurons); +for i =1:K + subplot(K,1,i) + plot(ACS(1).Cin(neurons(i),:)) +end + + diff --git a/BatchVerMO/centralA.m b/BatchVerMO/centralA.m new file mode 100644 index 0000000..172e5f6 --- /dev/null +++ b/BatchVerMO/centralA.m @@ -0,0 +1,8 @@ +function Atemp=centralA(Atemp) +for i=1:size(Atemp,2) + ai=Atemp(:,i); + temp = full(ai>quantile(ai, 0.5, 1)); + ai(~temp(:)) = 0; + Atemp(:,i)=ai; +end +end \ No newline at end of file diff --git a/BatchVerMO/cnmfeBatchVer_ClusterPart.m b/BatchVerMO/cnmfeBatchVer_ClusterPart.m new file mode 100644 index 0000000..27e1f9d --- /dev/null +++ b/BatchVerMO/cnmfeBatchVer_ClusterPart.m @@ -0,0 +1,107 @@ +%% C Run All below on cluster! + +%% 0. Get cluster ready +if running_on_cluster % some procedures making cluster use robust + [~, ~, ~] = maybe_spawn_workers(workersnum); + init_par_rng(2016); +end + +%% 1. Run normal CNMF-E for each file +File(length(samplelist)) = struct('options',[],'Ysignal',[]); % pre-allocate for parfor loop. +A0s=cell(1,length(samplelist)); +parfor i= 1:length(samplelist) + Mode='initiation'; + picname=samplelist(i).name %(namepattern) % For each file, save A's so you can roughly check what neuron is picked in which file. + name=fullfile(sampledir,samplelist(i).name); + [A0s{i},File(i)]=demo_endoscope2(gSig,gSiz,min_corr,min_pnr,min_pixel,bd,FS,SSub,TSub,bg_neuron_ratio,name,Mode,picname,[],File(i),convolveType,merge_thr); + fprintf('Sampling file number %.0f done\n', i); +end + +%%% delete samples that have no neurons. +emptyA0s_ind=find(cellfun('isempty', A0s)); +if ~isempty(emptyA0s_ind) + warning(['sample file number ',num2str(emptyA0s_ind),' with name below has/have no neuron extracted in it.\n']) + samplelist(emptyA0s_ind).name + fprintf('Deleting these sample A0s.'); + + samplelist(emptyA0s_ind)=[]; + A0s(emptyA0s_ind)=[]; + File(emptyA0s_ind)=[]; +end + +%%% Order similar neurons in the same sequence in each file, not necessary, +%%% but nice to do. It is fast. +A0s=Over_Days_ResequenceA(A0s,correlation_thresh,max2max2nd,skewnessthresh); + +%% 2. Next, Use this A, in each file i, find C's corresponding to each A's found in file j. +ACS(length(samplelist)) = struct('Ain',[],'Cin',[],'STD',[]); +S_R=length(samplelist); +parfor i= 1:S_R + Ain=[]; Cin=[]; STD=[]; + for j=1:S_R % parfor needs this + Aj=A0s{j}; + ACS_temp=A2C2A(File(i), Aj, File(i).options); + Ain = [Ain ACS_temp.Ain]; Cin = [Cin; ACS_temp.Cin]; STD=[STD ACS_temp.STD]; + end + ACS(i).Ain=Ain; ACS(i).Cin=Cin; ACS(i).STD=STD; +end + +%% 3 Merge similar neurons + +Amask_temp=cat(2,A0s{:}); +Amask_temp=bsxfun(@gt,Amask_temp,quantile(Amask_temp,0.3)); %only use central part for merging. +[Afinal,MC,newIDs,merged_ROIs] = mergeAC(Amask_temp,ACS,merge_thr_2); + +save([outputdir 'commonAcnmfeBatchVer.mat'],'-v7.3') + +%% 4 Determine Afinal that will be used to extract C's in each file. + +%%% Some processes making Afinal nicer, modified from Pengcheng Zhou's +%%% idea. +for i=1:size(Afinal,2) + ai=Afinal(:,i); + temp = full(ai>quantile(ai, 0.5, 1)); + ai(~temp(:)) = 0; + Afinal(:,i)=ai; +end + +% Just in case some all zero A's got passed to this stage. +nz_ind=any(Afinal); +Afinal=Afinal(:,nz_ind); +newIDs=newIDs(nz_ind); + + +Apicname=sprintf('%.0fAfinal',daynum); +ColorAllNeurons(Afinal,File(1).options.d1,File(2).options.d2,Apicname,Aoutputdir); + +Vars = {'Afinal';'samplelist';'File'}; Vars=Vars'; +eval(sprintf('save %s%0.f_cnmfe_BatchVer_ClusterPartI.mat %s -v7.3', Aoutputdir, daynum, strjoin(Vars))); +%% 5 "massive" procedure: Extract C from each file +neuron_batch(length(filelist)) = struct('ind_del',[],'signal',[],'FileOrigin',[],'neuron',[]); + +parfor i= 1:length(filelist) + mode='massive'; + nam=fullfile(datadir,filelist(i).name); + [~,neuron_batch(i)]=demo_endoscope2(gSig,gSiz,min_corr,min_pnr,min_pixel,bd,FS,SSub,TSub,bg_neuron_ratio,nam,mode,[],Afinal,neuron_batch(i),convolveType,merge_thr); + neuron_batch(i).FileOrigin=filelist(i); % save origin(filelist) +end +fprintf('Massive extraction in each file done.'); + +%% 6 Partition between those neurons found in each file and those not. Save results. + +parfor i= 1:length(filelist) + for j=1:size(neuron_batch(i).neuron.A,2) + jA=neuron_batch(i).neuron.A(:,j); + jC=neuron_batch(i).neuron.C(j,:); + neuron_batch(i).signal(j,:)=median(jA(jA>0)*jC); + end + fprintf('neuron_batch %.0f extraction done\n', i); +end + +%% 5.5 deconvolve signal +[~, ~, ~, ~,neuron_batch]=PartTraces(neuron_batch); + +%fprintf('First %.0f neurons are successfully deconvolved in each file while those after that are missing in some files\n', sum(~ind_del_final)); +fprintf('ALL extractions done.\n'); +eval(sprintf('save %sCNMFE_BatchVer.mat %s -v7.3', outputdir, 'neuron_batch')); +fprintf('ALL data saved, check them out!'); \ No newline at end of file diff --git a/BatchVerMO/cnmfeBatchVer_ClusterPartChuncked.m b/BatchVerMO/cnmfeBatchVer_ClusterPartChuncked.m new file mode 100644 index 0000000..2b63b1b --- /dev/null +++ b/BatchVerMO/cnmfeBatchVer_ClusterPartChuncked.m @@ -0,0 +1,61 @@ +%% C Run All below on cluster! +%% 0. Get cluster ready +if running_on_cluster % some procedures making cluster use robust + [~, ~, ~] = maybe_spawn_workers(workersnum); + init_par_rng(2016); +end + +%% 3 Merge similar neurons +%%% Merge similar neurons based on spatial AND temporal correlation +% C_all=cat(2,ACS.Cin); +% Amask_temp=cat(2,A0s{1:2}); +% Amask_temp=bsxfun(@gt,Amask_temp,quantile(Amask_temp,0.3)); %only use central part for merging. + +% C_temp=C_all(1:size(Amask_temp,2),:); +% [Amask_temp,C_temp,ACS] = mergeAC(Amask_temp,C_temp,ACS,merge_thr_2); +% +% merge2start=1+size(A0s{1},2)+size(A0s{2},2); +% for i=3:length(samplelist) +% Amask_temp=cat(2,Amask_temp,A0s{i}); +% Amask_temp=bsxfun(@gt,Amask_temp,quantile(Amask_temp,0.3)); +% +% C_temp=[C_temp;C_all(merge2start:merge2start+size(A0s{i},2)-1,:)]; +% +% [Amask_temp,C_temp,ACS] = mergeAC(Amask_temp,C_temp,ACS,merge_thr_2); +% merge2start=merge2start+size(A0s{i},2); +% end + +Amask_temp=cat(2,A0s{:}); +Amask_temp=bsxfun(@gt,Amask_temp,quantile(Amask_temp,0.3)); %only use central part for merging. +%C_all=cat(2,ACS.Cin); + +[Afinal,MC,newIDs,merged_ROIs] = mergeAC(Amask_temp,ACS,merge_thr_2); +% [size1,~]=cellfun(@size,newIDs); +% ACS(size1~=1) +save([outputdir 'commonAcnmfeBatchVer.mat'],'-v7.3') +%% 4 Collapsing A's +% As=cell(1,length(samplelist)); +% STDs=cell(1,length(samplelist)); +% parfor i=1:length(samplelist) +% As{i}=ACS(i).Ain; +% STDs{i}=ACS(i).STD; +% end +% Afinal=ReducingA(As,STDs); % the format for cell input is designed for potential other applications. +% save([outputdir 'uniqueAcnmfeBatchVer.mat'],'-v7.3') +%% 4.5 Determine Afinal that will be used to extract C's in each file. + +%%% Some processes making Afinal nicer, modified from Pengcheng Zhou's +%%% idea. +for i=1:size(Afinal,2) + ai=Afinal(:,i); + temp = full(ai>quantile(ai, 0.5, 1)); + ai(~temp(:)) = 0; + Afinal(:,i)=ai; +end + +% Just in case some all zero A's got passed to this stage. +nz_ind=any(Afinal); +Afinal=Afinal(:,nz_ind); +newIDs=newIDs(nz_ind); + +save([outputdir 'AfinalcnmfeBatchVer.mat'],'-v7.3') \ No newline at end of file diff --git a/BatchVerMO/cnmfeMoBatchVer_ClusterPart.m b/BatchVerMO/cnmfeMoBatchVer_ClusterPart.m new file mode 100644 index 0000000..aaa7596 --- /dev/null +++ b/BatchVerMO/cnmfeMoBatchVer_ClusterPart.m @@ -0,0 +1,146 @@ +%% C Run All below on cluster! +if strcmp(Version,'MoBatchVer') + fprintf('Running BatchVer with Motion Correction Later.') +elseif strcmp(Version,'BatchVer') + fprintf('Simple BatchVer') +end +if ~exist(outputdirDetails,'dir') + mkdir(outputdirDetails) +end +cd(outputdirDetails) + +%% 0. Get cluster ready +if running_on_cluster % some procedures making cluster use robust + [~, ~, ~] = maybe_spawn_workers(workersnum); + init_par_rng(2016); +end + +%% 1. Run normal CNMF-E for each file +File(length(samplelist)) = struct('options',[],'Ysignal',[],'neuron',[],'Ybg',[]); % Ysignal and Ybg for video making. + % pre-allocate for parfor loop. +A0s=cell(1,length(samplelist)); +parfor i= 1:length(samplelist) + Mode='initiation'; + picname=samplelist(i).name %(namepattern) % For each file, save A's so you can roughly check what neuron is picked in which file. + name=fullfile(sampledir,samplelist(i).name); + [A0s{i},File(i)]=demo_endoscope2(bg_neuron_ratio,merge_thr,with_dendrites,K,sframe,num2read,... + name,neuron_full,Mode,picname,File(i),[],... + thresh_detecting_frames); + fprintf('Sampling file number %.0f done\n', i); +end + +%%% delete samples that have no neurons. +emptyA0s_ind=find(cellfun('isempty', A0s)); +if ~isempty(emptyA0s_ind) + warning(['sample file number ',num2str(emptyA0s_ind),' with name below has/have no neuron extracted in it.\n']) + samplelist(emptyA0s_ind).name + fprintf('Deleting these sample A0s.'); + + samplelist(emptyA0s_ind)=[]; + A0s(emptyA0s_ind)=[]; + File(emptyA0s_ind)=[]; +end + +%%% Order similar neurons in the same sequence in each file, not necessary, +%%% but nice to do. It is fast. +A0s=Over_Days_ResequenceA(A0s,correlation_thresh,max2max2nd,skewnessthresh); +save([outputdirDetails 'EachFilecnmfeBatchVer.mat'],'-v7.3') + +if strcmp(Version,'MoBatchVer') + eval(sprintf('save %s%0.f_cnmfe_BatchVer_PartI_File.mat %s -v7.3', outputdir, daynum, 'File')); + fprintf('Variable ''File'', which includes fields of ''options'' ''Ysignal'' ''neuron'' ''Ybg'' saved.'); +elseif strcmp(Version,'BatchVer') + Vars = {'File'}; Vars=Vars'; + eval(sprintf('save %s%0.f_cnmfe_BatchVer_ClusterPartI.mat %s -v7.3', outputdir, daynum, strjoin(Vars))); +end +File = rmfield(File,{'Ybg','neuron'}); + +%% 2. Next, Use this A, in each file i, find C's corresponding to each A's found in file j. +ACS(length(samplelist)) = struct('Cin',[],'Cin_raw',[],'STD',[]); +S_R=length(samplelist); +parfor i= 1:S_R + Cin=[]; Cin_raw=[]; STD=[]; + for j=1:S_R % parfor needs S_R rather than length(samplelist) + Aj=A0s{j}; + ACS_temp=A2C(File(i).Ysignal, Aj, File(i).options); + Cin = [Cin; ACS_temp.Cin]; STD=[STD ACS_temp.STD]; Cin_raw=[Cin_raw; ACS_temp.Cin_raw]; + end + ACS(i).Cin=Cin; ACS(i).Cin_raw=Cin_raw; ACS(i).STD=STD; +end +save([outputdirDetails 'ACScnmfeBatchVer.mat'],'-v7.3') + +%% 3 Merge similar neurons + +A_temp=cat(2,A0s{:}); +%Amask_temp=bsxfun(@gt,Amask_temp,quantile(Amask_temp,0.3)); %only use central part for merging. +[Afinal,MC,newIDs,merged_ROIs,close_ind,real_ind] = mergeAC(A_temp,ACS,merge_thr_2,dmin,File(1).options.d1,File(1).options.d2); + +if strcmp(Version,'MoBatchVer'); save([outputdirDetails 'commonAcnmfeBatchVer.mat'],'-v7.3') +elseif strcmp(Version,'BatchVer');save([outputdir 'commonAcnmfeBatchVer.mat'],'-v7.3') +end + +%% 4 Determine Afinal that will be used to extract C's in each file. + +%%% Some processes making Afinal nicer, modified from Pengcheng Zhou's +%%% idea. +for i=1:size(Afinal,2) + ai=Afinal(:,i); + temp = full(ai>quantile(ai, 0.5, 1)); + ai(~temp(:)) = 0; + Afinal(:,i)=ai; +end + +% Just in case some all zero A's got passed to this stage. +nz_ind=any(Afinal); +Afinal=Afinal(:,nz_ind); +newIDs=newIDs(nz_ind); + + +Apicname=sprintf('%.0fAfinal',daynum); +if strcmp(Version,'MoBatchVer') + ColorAllNeurons(Afinal,File(1).options.d1,File(1).options.d2,[Apicname ' PNR=' num2str(neuron_full.options.min_pnr)],outputdirDetails); + Vars = {'Afinal';'samplelist'}; Vars=Vars'; + eval(sprintf('save %s%0.f_cnmfe_BatchVer_PartI_Afinalsam.mat %s -v7.3', outputdir, daynum, strjoin(Vars))); + fprintf('cnmfe_BatchVer_for motion Part1 data saved, check them out!'); + return +elseif strcmp(Version,'BatchVer') + ColorAllNeurons(Afinal,File(1).options.d1,File(1).options.d2,Apicname,outputdir); + Vars = {'Afinal';'samplelist'}; Vars=Vars'; + eval(sprintf('save %s%0.f_cnmfe_BatchVer_ClusterPartI.mat %s -append', outputdir, daynum, strjoin(Vars))); +end + +% The following will be executed for cnmf_e(BatchVer), without motion +% correction. +%% 5 "massive" procedure: Extract A from each file +neuron_batch(length(samplelist)) = struct('ind_del',[],'rawsignal',[],'signal',[],'DeconvSpiketrain',[],'FileOrigin',[],'neuron',[],'C',[],'C_raw',[]); + +parfor i= 1:length(samplelist) + mode='massive'; + nam=cell(1,2); + nam{1}=fullfile(datadir,samplelist(i).name); + nam{2}=File(i).Ysignal; + [~,neuron_batch(i)]=demo_endoscope2(bg_neuron_ratio,[],with_dendrites,K,sframe,num2read,... + nam,neuron_full,mode,[],neuron_batch(i),Afinal,... + thresh_detecting_frames); + neuron_batch(i).FileOrigin=filelist(i); % save origin(filelist) +end +neuron_batch = rmfield(neuron_batch,{'C','C_raw'}); +fprintf('Massive extraction in each file done.'); + +%% 6 Save A*C + +parfor i= 1:length(samplelist) + for j=1:size(neuron_batch(i).neuron.A,2) + jA=neuron_batch(i).neuron.A(:,j); + jC=neuron_batch(i).neuron.C_raw(j,:); + neuron_batch(i).rawsignal(j,:)=median(jA(jA>0)*jC); + end + fprintf('neuron_batch %.0f extraction done\n', i); +end + +%% 7 deconvolve signal +[~, ~, ~, ~,~,neuron_batch,boundary_raw]=PartTraces(neuron_batch); +Vars = {'neuron_batch';'boundary_raw'}; Vars=Vars'; +fprintf('ALL extractions done.\n'); +eval(sprintf('save %sCNMFE_BatchVer.mat %s -v7.3', outputdir, strjoin(Vars))); +fprintf('ALL data saved, check them out!'); diff --git a/BatchVerMO/cnmfeMoBatchVer_ClusterPart2.m b/BatchVerMO/cnmfeMoBatchVer_ClusterPart2.m new file mode 100644 index 0000000..b2ae998 --- /dev/null +++ b/BatchVerMO/cnmfeMoBatchVer_ClusterPart2.m @@ -0,0 +1,156 @@ +%% E Run All below on cluster! + +% % cnmfefolder is where 1)logistics,2)PartI's results,3)Motion-corrected A's are saved +% cnmfefolder='/net/feevault/data0/shared/EmilyShijieShared_old/6922_moBatchVerNYVersion/'; +% +% % load one of the logistics +% load(fullfile(cnmfefolder,'LogisticscnmfeBatchVer20170713.mat')); +% +% % load motion corrected A's +% load(fullfile(cnmfefolder,'cnmfe_BatchVer_PartII_MotionCorrection.mat')) +M=M_GUI; +cnmfefolder=outputdir; +%--------------------------No need to read below------------------------------- +%% 0. Get cluster ready +if running_on_cluster % some procedures making cluster use robust + [~, ~, ~] = maybe_spawn_workers(workersnum); + init_par_rng(2016); +end +%% 1. load samplelist,A and sample's File from ClusterI into big structures +AandSample_list=dir(fullfile(cnmfefolder,'*PartI_Afinalsam*')); +Filesignal_list=dir(fullfile(cnmfefolder,'*PartI_File*')); + +eachdayfilenum=[]; +for i=1:numel(AandSample_list) %go through days + File_temponeday=load(fullfile(cnmfefolder,Filesignal_list(i).name)); + AandSample_temponeday=load(fullfile(cnmfefolder,AandSample_list(i).name)); + eachdayfilenum=[eachdayfilenum length(File_temponeday.File)]; + clear File_temp + File_temp(length(File_temponeday.File)) = struct('Ysignal',[],'options',[]); + for j=1:length(File_temponeday.File) + File_temp(j).Ysignal=File_temponeday.File(j).Ysignal; + File_temp(j).options=File_temponeday.File(j).options; + end + clear File_temponeday + if i==1 + filelist_fulllist=AandSample_temponeday.samplelist; + File_fulllist=File_temp; + else + filelist_fulllist=[filelist_fulllist; AandSample_temponeday.samplelist]; + File_fulllist=[File_fulllist File_temp]; + end +end +%-------------------Not needed 99% of situations------------------- +%%%%% File_samplelist can be used to substitute for File_fulllist if there +%%%%% are too many files. + daylength=length(eachdayfilenum); avefilenum=mean(eachdayfilenum); +every_file_num=floor(daylength^2/avefilenum); +choose_ind=mod(1:numel(filelist_fulllist),every_file_num)==1; +filelist_samplelist=filelist_fulllist(choose_ind); +File_samplelist=File_fulllist(choose_ind); +%----------------------Just Leave them here------------------------- + +%%% Order similar neurons in the same sequence in each file, not necessary, +%%% but nice to do. It is fast. +M1=Over_Days_ResequenceAForMo(M,correlation_thresh,max2max2nd,skewnessthresh); + +%% 2. Next, Use M1, in each file i, find C's corresponding to each A's found in file j. +eachfilenum_cumsum=cumsum(eachdayfilenum); +filenumsum = eachfilenum_cumsum(end); +ACS(filenumsum) = struct('Cin',[],'Cin_raw',[],'STD',[]); + +S_L=length(eachfilenum_cumsum); + +parfor i= 1:filenumsum % file + Cin=[]; Cin_raw=[]; STD=[]; + k=find((eachfilenum_cumsum>=i),1); + for j=1:S_L % parfor needs this + Aj=M1{k}{j}; + ACS_temp=A2C(File_fulllist(i).Ysignal, Aj, File_fulllist(i).options); + Cin = [Cin; ACS_temp.Cin]; Cin_raw=[Cin_raw; ACS_temp.Cin_raw]; STD=[STD ACS_temp.STD]; + end + ACS(i).Cin=Cin; ACS(i).Cin_raw=Cin_raw; ACS(i).STD=STD; +end +save([outputdir 'PartTwoOFcnmfeBatchVerMOTION.mat'],'-v7.3') +%% 3 Merge similar neurons +%%% Merge similar neurons based on spatial AND temporal correlation +% use the highest correlation one, here we use the middle day one to approximate. + +midone=round(S_L/2); +Amask_temp=cat(2,M1{midone}{:}); + +%Amask_temp=bsxfun(@gt,Amask_temp,quantile(Amask_temp,0.3)); %only use central part for merging. +C=cat(2,ACS.Cin); +d1=File_fulllist(1).options.d1; +d2=File_fulllist(1).options.d2; +dmin=4;%%%%%%%%%% +clear ACS File_samplelist %File_fulllist +[M2,MC,newIDs,merged_ROIs,close_ind,eachday_ind] = mergeACforMo(Amask_temp,C,merge_thr_2,M1,dmin,d1,d2); + +save([outputdir 'RoughAfinalcnmfeBatchVerMOTION.mat'],'-v7.3') +%% Print Log file for regitration of neurons. + +%% 4 Determine Afinal that will be used to extract C's in each file. + +%%% Some processes making Afinal nicer, modified from Pengcheng Zhou's +%%% idea. +M3=cell(1,numel(M2)); +for c=1:numel(M2) + Afinal=M2{c}; + for i=1:size(Afinal,2) + ai=Afinal(:,i); + temp = full(ai>quantile(ai, 0.5, 1)); + ai(~temp(:)) = 0; + Afinal(:,i)=ai; + end + % Just in case some all zero A's got passed to this stage. + nz_ind=any(Afinal); + Afinal=Afinal(:,nz_ind); + if c==1 + newIDs=newIDs(nz_ind); + end + Apicname=sprintf('Day%.0fAFinal',num2str(totaldays(c))); + ColorAllNeurons(Afinal,d1,d2,Apicname,[outputdir, num2str(totaldays(c)), '/']); + M3{c}=Afinal; +end +Vars = {'newIDs';'close_ind';'M2';'M3'}; Vars=Vars'; +eval(sprintf('save %sAfinalcnmfeBatchVerMotion %s -v7.3', outputdir, strjoin(Vars))); +save([outputdir 'NiceAfinalcnmfeBatchVerMOTION.mat'],'-v7.3') + +%% 5 "massive" procedure: Extract A from each file +neuron_batchMO(length(filelist_fulllist)) = struct('ind_del',[],'rawsignal',[],'signal',[],'DeconvSpiketrain',[],'FileOrigin',[],'neuron',[],'C',[],'C_raw',[]); + +parfor i= 1:length(filelist_fulllist) + mode='massive'; + nam=cell(1,2); + nam{1}=fullfile(datadir,filelist_fulllist(i).name); + nam{2}=File_fulllist(i).Ysignal; + k=find((eachfilenum_cumsum>=i),1); + [~,neuron_batchMO(i)]=demo_endoscope2(bg_neuron_ratio,[],with_dendrites,K,sframe,num2read,... + nam,neuron_full,mode,[],neuron_batchMO(i),M3{k},... + thresh_detecting_frames); + + neuron_batchMO(i).FileOrigin=filelist_fulllist(i); % save origin(filelist) +end +neuron_batchMO = rmfield(neuron_batchMO,{'C','C_raw'}); +fprintf('Massive extraction done.'); +save([outputdir 'MassivecnmfeBatchVerMotion.mat'],'-v7.3') + +%% 6 Save results. + +for i= 1:length(filelist_fulllist) + for j=1:size(neuron_batchMO(i).neuron.A,2) + jA=neuron_batchMO(i).neuron.A(:,j); + jC=neuron_batchMO(i).neuron.C_raw(j,:); + neuron_batchMO(i).rawsignal(j,:)=median(jA(jA>0)*jC); + end + fprintf('neuron_batch %.0f extraction done\n', i); +end + +%% 7 deconvolve signal +[~, ~, ~, ~,~,neuron_batchMO,boundary_raw]=PartTraces(neuron_batchMO); +Vars = {'neuron_batchMO';'boundary_raw'}; Vars=Vars'; + +fprintf('ALL moBatchVer extractions done.\n'); +eval(sprintf('save %sCNMFE_moBatchVer.mat %s -v7.3', outputdir, strjoin(Vars))); +fprintf('ALL moBatchVer data saved, check them out!'); \ No newline at end of file diff --git a/BatchVerMO/cnmfeMoBatchVer_ClusterPart2Trunked.m b/BatchVerMO/cnmfeMoBatchVer_ClusterPart2Trunked.m new file mode 100644 index 0000000..645c5ee --- /dev/null +++ b/BatchVerMO/cnmfeMoBatchVer_ClusterPart2Trunked.m @@ -0,0 +1,73 @@ +%% C-2 Run All below on cluster! + +%% 0. Get cluster ready +if running_on_cluster % some procedures making cluster use robust + [~, ~, ~] = maybe_spawn_workers(workersnum); + init_par_rng(2016); +end +load(fullfile(outputdir,'RoughAfinalcnmfeBatchVerMOTION.mat')) + +%% 4 Determine Afinal that will be used to extract C's in each file. + +%%% Some processes making Afinal nicer, modified from Pengcheng Zhou's +%%% idea. +M3=cell(1,numel(M2)); +for c=1:numel(M2) + Afinal=M2{c}; + for i=1:size(Afinal,2) + ai=Afinal(:,i); + temp = full(ai>quantile(ai, 0.5, 1)); + ai(~temp(:)) = 0; + Afinal(:,i)=ai; + end + % Just in case some all zero A's got passed to this stage. + nz_ind=any(Afinal); + Afinal=Afinal(:,nz_ind); + if c==1 + newIDs=newIDs(nz_ind); + end + Apicname=sprintf('Day%.0fAFinal',num2str(totaldays(c))); + ColorAllNeurons(Afinal,d1,d2,Apicname,[outputdir, num2str(totaldays(c)), '/']); + M3{c}=Afinal; +end +Vars = {'newIDs';'close_ind';'M2';'M3'}; Vars=Vars'; +eval(sprintf('save %sAfinalcnmfeBatchVerMotion %s -v7.3', outputdir, strjoin(Vars))); +save([outputdir 'NiceAfinalcnmfeBatchVerMOTION.mat'],'-v7.3') + +%% 5 "massive" procedure: Extract A from each file +neuron_batchMO(length(filelist_fulllist)) = struct('ind_del',[],'rawsignal',[],'signal',[],'FileOrigin',[],'neuron',[],'C',[],'C_raw',[]); + +parfor i= 1:length(filelist_fulllist) + mode='massive'; + nam=cell(1,2); + nam{1}=fullfile(datadir,filelist_fulllist(i).name); + nam{2}=File_fulllist(i).Ysignal; + k=find((eachfilenum_cumsum>=i),1); + [~,neuron_batchMO(i)]=demo_endoscope2(bg_neuron_ratio,[],with_dendrites,K,sframe,num2read,... + nam,neuron_full,mode,[],neuron_batchMO(i),M3{k},... + thresh_detecting_frames); + + neuron_batchMO(i).FileOrigin=filelist_fulllist(i); % save origin(filelist) +end +neuron_batchMO = rmfield(neuron_batchMO,{'C','C_raw'}); +fprintf('Massive extraction done.'); +save([outputdir 'MassivecnmfeBatchVerMotion.mat'],'-v7.3') + +%% 6 Save results. + +for i= 1:length(filelist_fulllist) + for j=1:size(neuron_batchMO(i).neuron.A,2) + jA=neuron_batchMO(i).neuron.A(:,j); + jC=neuron_batchMO(i).neuron.C_raw(j,:); + neuron_batchMO(i).rawsignal(j,:)=median(jA(jA>0)*jC); + end + fprintf('neuron_batch %.0f extraction done\n', i); +end + +%% 7 deconvolve signal +[~, ~, ~, ~,~,neuron_batchMO,boundary_raw]=PartTraces(neuron_batchMO); +Vars = {'neuron_batchMO';'boundary_raw'}; Vars=Vars'; + +fprintf('ALL moBatchVer extractions done.\n'); +eval(sprintf('save %sCNMFE_moBatchVer.mat %s -v7.3', outputdir, strjoin(Vars))); +fprintf('ALL moBatchVer data saved, check them out!'); \ No newline at end of file diff --git a/BatchVerMO/cnmfeMoBatchVer_ClusterPart_Trunked.m b/BatchVerMO/cnmfeMoBatchVer_ClusterPart_Trunked.m new file mode 100644 index 0000000..5473318 --- /dev/null +++ b/BatchVerMO/cnmfeMoBatchVer_ClusterPart_Trunked.m @@ -0,0 +1,41 @@ +%% C Run All below on cluster! + +%% 0. Get cluster ready +if running_on_cluster % some procedures making cluster use robust + [~, ~, ~] = maybe_spawn_workers(workersnum); + init_par_rng(2016); +end + +%% 5 "massive" procedure: Extract A from each file +neuron_batch(length(samplelist)) = struct('ind_del',[],'rawsignal',[],'signal',[],'DeconvSpiketrain',[],'FileOrigin',[],'neuron',[],'C',[],'C_raw',[]); + +parfor i= 1:length(samplelist) + mode='massive'; + nam=cell(1,2); + nam{1}=fullfile(datadir,samplelist(i).name); + nam{2}=File(i).Ysignal; + [~,neuron_batch(i)]=demo_endoscope2(bg_neuron_ratio,[],with_dendrites,K,sframe,num2read,... + nam,neuron_full,mode,[],neuron_batch(i),Afinal,... + thresh_detecting_frames); + neuron_batch(i).FileOrigin=filelist(i); % save origin(filelist) +end +neuron_batch = rmfield(neuron_batch,{'C','C_raw'}); +fprintf('Massive extraction in each file done.'); + +%% 6 Save A*C + +parfor i= 1:length(samplelist) + for j=1:size(neuron_batch(i).neuron.A,2) + jA=neuron_batch(i).neuron.A(:,j); + jC=neuron_batch(i).neuron.C_raw(j,:); + neuron_batch(i).rawsignal(j,:)=median(jA(jA>0)*jC); + end + fprintf('neuron_batch %.0f extraction done\n', i); +end + +%% 7 deconvolve signal +[~, ~, ~, ~,~,neuron_batch,boundary_raw]=PartTraces(neuron_batch); +Vars = {'neuron_batch';'boundary_raw'}; Vars=Vars'; +fprintf('ALL extractions done.\n'); +eval(sprintf('save %sCNMFE_BatchVer.mat %s -v7.3', outputdir, strjoin(Vars))); +fprintf('ALL data saved, check them out!'); \ No newline at end of file diff --git a/BatchVerMO/cnmfe_update_BG_2.m b/BatchVerMO/cnmfe_update_BG_2.m new file mode 100644 index 0000000..cdcef17 --- /dev/null +++ b/BatchVerMO/cnmfe_update_BG_2.m @@ -0,0 +1,23 @@ +clear Ysignal; +tic; +display(size(NEURON.A)) +display(size(NEURON.C)) +display(size(Y)) +Ybg = Y-NEURON.A*NEURON.C; +rr = ceil(NEURON.options.gSiz * bg_neuron_ratio); +active_px = []; %(sum(IND, 2)>0); %If some missing neurons are not covered by active_px, use [] to replace IND +[Ybg, Ybg_weights] = NEURON.localBG(Ybg, spatial_ds_factor, rr, active_px, NEURON.P.sn, thresh); % estiamte local background. +% subtract the background from the raw data. +Ysignal = Y - Ybg; + +%% estimate noise +if ~isfield(NEURON.P, 'sn') || isempty(NEURON.P.sn) + %% estimate the noise for all pixels + b0 =zeros(size(Ysignal,1), 1); + sn = b0; + parfor m=1:size(NEURON.A,1) + [b0(m), sn(m)] = estimate_baseline_noise(Ysignal(m, :)); + end + Ysignal = bsxfun(@minus, Ysignal, b0); + NEURON.P.sn = sn; +end \ No newline at end of file diff --git a/BatchVerMO/corr_ForBatch.m b/BatchVerMO/corr_ForBatch.m new file mode 100644 index 0000000..b9f5401 --- /dev/null +++ b/BatchVerMO/corr_ForBatch.m @@ -0,0 +1,87 @@ +%% Making correlation images from Y +function corr_ForBatch(outputdir,datadir,d1,d2) +%outputdir='/net/feevault/data0/shared/EmilyShijieShared_old/6922_moBatchVerNYVersion/'; +outputdir_Corr=[outputdir 'Correlation/']; + +if ~exist(outputdir_Corr,'dir') + mkdir(outputdir_Corr) +end +cd(outputdir_Corr) + +log_list=dir(fullfile(outputdir,'*LogisticscnmfeBatchVer*')); +display(length(log_list)) +for i=1:length(log_list) + log_nam=log_list(1).name; + load(fullfile(outputdir,log_nam)) + + Y=[]; + for j=1:length(filelist) + Y_tmp=matfile(fullfile(datadir,filelist(j).name)); + Y=cat(3,Y,Y_tmp.Y); + end + display(size(Y)) + Y=double(Y); + Y(isnan(Y)) = 0; % remove nan values + + % divide data into multiple patches + patch_sz = [3, 3]; + r0_patch = round(linspace(1, d1, 1+patch_sz(1))); + c0_patch = round(linspace(1, d2, 1+patch_sz(2))); + nr_patch = length(r0_patch)-1; + nc_patch = length(c0_patch)-1; + Cn = zeros(d1, d2); + PNR = zeros(d1,d2); + % compute correlation_image patch by patch + bd = 20; + sig=5; % thresholding noise by sig*std() + for mr = 1:nr_patch + r0 = max(1, r0_patch(mr)-bd); % minimum row index of the patch + r1 = min(d1, r0_patch(mr+1)+bd-1); % maximum row index of the patch + for mc = 1:nc_patch + c0 = max(1, c0_patch(mc)-bd); % minimum column index of the patch + c1 = min(d2, c0_patch(mc+1)+bd-1); % maximum column index of the patch + + % take the patch from the raw data + nrows = (r1-r0+1); % number of rows in the patch + ncols = (c1-c0+1); %number of columns in the patch + Ypatch = double(Y(r0:r1, c0:c1, :)); + + % spatially filter the data + %HY = imfilter(Ypatch, psf, 'replicate'); + + % copute signal to noise ratio + %HY = reshape(HY, [], T); + %HY = bsxfun(@minus, HY, median(HY, 2)); + HY_max=max(Ypatch,[],2);%HY_max = max(HY, [], 2); + Ysig = get_noise_fft(Ypatch); +% tmp_PNR = reshape(HY_max./Ysig, nrows, ncols); +% PNR(r0:r1, c0:c1) = max(PNR(r0:r1, c0:c1), tmp_PNR); + + + % compute loal correlation + Ypatch(bsxfun(@lt, Ypatch, Ysig*sig)) = 0; + tmp_Cn = correlation_image(Ypatch, [1,2], nrows, ncols); + Cn(r0:r1, c0:c1) = max(Cn(r0:r1, c0:c1), tmp_Cn); + end + end + figure + imagesc(Cn, [0, 1]); colorbar; + axis equal off tight; + fignam=['correlation image of ' num2str(daynum)]; + title(fignam); + saveas(gcf,fignam); +end +% Filesignal_list=dir(fullfile(outputdir,'*PartI_File*')); +% for i=1:length(Filesignal_list) +% nam=Filesignal_list(1).name; +% daynum=nam(1:8); +% load(fullfile(outputdir,['LogisticscnmfeBatchVer' nam(1:8) '.mat'])) +% load(fullfile(outputdir,nam)) +% +% d1=File(1).options.d1; d2=File(1).options.d2; +% MakingVideos(File,d1,d2,num2str(daynum),outputdir_video) +% clear File +% MakingVideos([],d1,d2,num2str(daynum),outputdir_video,true,datadir,filelist) +% fprintf([daynum 'videos saved, check them out!']); +% end +fprintf('ALL correlation images saved, check them out!'); \ No newline at end of file diff --git a/BatchVerMO/demo_batched_version.asv b/BatchVerMO/demo_batched_version.asv new file mode 100644 index 0000000..d84d1e1 --- /dev/null +++ b/BatchVerMO/demo_batched_version.asv @@ -0,0 +1,164 @@ +%% Batched CNMF-E version +% Shijie Gu, techel@live.cn + +% Direction of use: +% Input in Section1, let other sections run automatically. +%% 1 Input +% (1) code directory, data directory, how many every files to sample. +% (2) normal CNMF-E parameters. + +feevault +codeDir='/home/elm/CaImagingCode/'; +addpath(genpath(codeDir)); +datadir='/net/feevault/data0/elm/ProcessedCalciumData/7030FirstFewDaysForBatch'; %%%%% +outputdir=datadir; + +kind='*CaELM*'; +Datadir=[datadir,kind]; +filelist=dir(Datadir); +every_file_num=5; % choose final A from every every_file_num as samples in the folders. + +running_on_cluster=true; % running on cluster or not + +gSig=10; +gSiz=17; +min_pnr=6.5; +bg_neuron_ratio = 1; % spatial range / diameter of neurons +% also "picname" in section2 should be catered to your way of naming files. + +if running_on_cluster + [poolObj, ownPool, poolSize] = maybe_spawn_workers(4); + init_par_rng(2016); +end +%% 2 +%%% Preparation for getting A from each sample file. +if sum(cellfun('isempty', {filelist.date}))>0 + display(['error in folder, might be an empty folder:',datadir]) + return +else + filelist=filelist(mod(1:numel(filelist),every_file_num)==1); % pick every some files in the folder as our sample for estimating A. +end + +% pre-allocate for parfor loop. +%File is the main output structure of demo_endoscope2. It is different for +%sampling process (mode="initiation") and for running for each file with a given +%A, (mode="massive") + % containing each file's Y or Ysignal which is denoised and background + % subtracted data. +%A0s is the output structure containing all A's obtained from each of the + %sample files. + +File(length(filelist)) = struct('options',[],'Y',[],'Ysignal',[]); +A0s=cell(1,length(filelist)); + +%%% Running normal CNMF-E for each file +parfor i= 1:length(filelist) + picname=filelist(i).name(1:35) % For each file, save A's so you can roughly check what neuron is picked in which file. + name=fullfile(datadir,filelist(i).name); + % The new cnmf-e demo is converted to + % function to cater to parfor loop. + % Meanwhile, some part of cnmf-e demo + % is applied to this sampling process + % while some simplied methods are + % applied for later for each file. + % Use 'mode' variable to specify this. + mode='initiation'; + [A0s{i},File(i)]=demo_endoscope2(gSig,gSiz,min_pnr,bg_neuron_ratio,name,mode,picname,[],File(i)); + fprintf('Sampling file number %.0f done\n', i); +end +%% +%%% Order similar neurons in the same sequence in each file, not necessary, +%%% but nice to do. It is fast. +[ns_storage_1]=Over_Days_findAnn(A0s,0.6,1.1,0); +for i= 1:length(filelist) + A0=A0s{i}; + corr_ind=ns_storage_1(:,1); + unique_ind=setdiff(1:size(A0,2),corr_ind); + A0s{i}=[A0(:,corr_ind) A0(:,unique_ind)]; +end + +%%% This A is raw and plain, it is just all A's from all files concatnated +%%% together. +A=cat(2,A0s{:}); +Amask=A>0; + +%%% Use this A, in each file i, find C's corresponding to each A's found in +%%% file j. +ACS(length(filelist)) = struct('Ain',[],'Cin',[],'STD',[]); + +parfor i= 1:length(filelist) + for j=1:length(filelist) + ACS(i)=A2C2A(ACS(i),File(i), A0s, j, File(i).options, 1); + end +end +%% 3 Determine commonA's +%%% Merge similar neurons based on spatial AND temporal correlation +merge_thr=[0.7,0.7]; +[merged_ROIs,commonA,commonC,ind_del] = mergeAC(Amask,ACS,merge_thr); + +%% 4 Determine uniqueA's +Aunique=cell(1,length(filelist)); +STDunique=cell(1,length(filelist)); +parfor i=1:length(filelist) + FileA=ACS(i).Ain; + FileSTD=ACS(i).STD; + Aunique{i}=FileA(:,~ind_del); + STDunique{i}=FileSTD(~ind_del); +end +weightedA=ReducingA(Aunique,STDunique); + +%% 5 Determine Afinal that will be used to extract C's in each file. +Afinal=cat(2,commonA,weightedA); + +%%% Some processes making Afinal nicer, modified from Pengcheng Zhou's +%%% idea. +for i=1:size(Afinal,2) + ai=Afinal(:,i); + temp = full(ai>quantile(ai, 0.5, 1)); +% l = bwlabel(reshape(temp, File(1).options.d1, File(1).options.d2), 4); +% temp(l~=l(ind_ctr)) = false; + ai(~temp(:)) = 0; + Afinal(:,i)=ai; +end + +nz_ind=any(Afinal); +Afinal=Afinal(:,nz_ind); + +%% 6 "massive" procedure: Extract A from each file +filelist=dir(Datadir); +FILE(length(filelist)) = struct('A',[],'C',[],'ind_del',[]); +parfor i= 1:length(filelist) + picname=filelist(i).name(1:35) + nam=fullfile(datadir,filelist(i).name); + mode='massive'; + [~,FILE(i)]=demo_endoscope2(gSig,gSiz,min_pnr,bg_neuron_ratio,nam,mode,picname,Afinal,[]); +end +fprintf('Main extraction done'); + +neuron(length(filelist)) = struct('signal',[],'filelist',[]); +%%% Partition between those neurons found in each file and those not. +ind_del_final_cat=cat(2,FILE.ind_del); +ind_del_final=any(ind_del_final_cat,2); +parfor i= 1:length(filelist) + nA=FILE(i).A; + FILE(i).A=[nA(:,~ind_del_final) nA(:,ind_del_final)]; + fprintf('A extraction done'); + nC=FILE(i).C; + FILE(i).C=[nC(~ind_del_final,:);nC(ind_del_final,:)]; + fprintf('C extraction done'); + %%% save this data's origin + + for j=1:size(FILE(i).A,2) + jA=FILE(i).A(:,j); + jC=FILE(i).C(j,:); + neuron(i).signal(j,:)=median(jA(jA>0)*jC); + neuron(i).filelist=filelist(i); + end + fprintf('FILE %.0f extraction done\n', i); +end +fprintf('First %.0f neurons are found in each files while those after that are missing in some files', sum(ind_del_final)); +fprintf('ALL extraction done'); +eval(sprintf('save %sCNMFE_%s.mat %s -v7.3', outputdir, kind, 'neuron FILE')); + +%% 7 Post-analysis + diff --git a/BatchVerMO/demo_batched_version.m b/BatchVerMO/demo_batched_version.m new file mode 100644 index 0000000..7747c39 --- /dev/null +++ b/BatchVerMO/demo_batched_version.m @@ -0,0 +1,137 @@ +%% Batched CNMF-E version +% Shijie Gu, techel@live.cn + +% Direction of use: +% Input in Section1, let other sections run automatically. + +gSig=10; +gSiz=17; +min_pnr=6.5; +bg_neuron_ratio = 1; % spatial range / diameter of neurons +% also "picname" in section2 should be catered to your way of naming files. + +%% 2 +% pre-allocate for parfor loop. +%File is the main output structure of demo_endoscope2. It is different for +%sampling process (mode="initiation") and for running for each file with a given +%A, (mode="massive") + % containing each file's Y or Ysignal which is denoised and background + % subtracted data. +%A0s is the output structure containing all A's obtained from each of the + %sample files. + +File(length(filelist)) = struct('options',[],'Y',[],'Ysignal',[]); +A0s=cell(1,length(filelist)); + +%%% Running normal CNMF-E for each file +parfor i= 1:length(filelist) + picname=filelist(i).name(1:35) % For each file, save A's so you can roughly check what neuron is picked in which file. + name=fullfile(datadir,filelist(i).name); + % The new cnmf-e demo is converted to + % function to cater to parfor loop. + % Meanwhile, some part of cnmf-e demo + % is applied to this sampling process + % while some simplied methods are + % applied for later for each file. + % Use 'mode' variable to specify this. + mode='initiation'; + [A0s{i},File(i)]=demo_endoscope2(gSig,gSiz,min_pnr,bg_neuron_ratio,name,mode,picname,[],File(i)); + fprintf('Sampling file number %.0f done\n', i); +end + +%%% Order similar neurons in the same sequence in each file, not necessary, +%%% but nice to do. It is fast. +[ns_storage_1]=Over_Days_findAnn(A0s,0.6,1.1,0); +for i= 1:length(filelist) + A0=A0s{i}; + corr_ind=ns_storage_1(:,1); + unique_ind=setdiff(1:size(A0,2),corr_ind); + A0s{i}=[A0(:,corr_ind) A0(:,unique_ind)]; +end + +%%% This A is raw and plain, it is just all A's from all files concatnated +%%% together. +A=cat(2,A0s{:}); +Amask=A>0; + +%%% Use this A, in each file i, find C's corresponding to each A's found in +%%% file j. +ACS(length(filelist)) = struct('Ain',[],'Cin',[],'STD',[]); + +parfor i= 1:length(filelist) + for j=1:length(filelist) + Aj=A0s{j}; + ACS(i)=A2C2A(ACS(i), File(i), Aj, File(i).options); + end +end +%% 3 Determine commonA's +%%% Merge similar neurons based on spatial AND temporal correlation +merge_thr=[0.7,0.7]; +[merged_ROIs,commonA,commonC,ind_del] = mergeAC(Amask,ACS,merge_thr); + +%% 4 Determine uniqueA's +Aunique=cell(1,length(filelist)); +STDunique=cell(1,length(filelist)); +parfor i=1:length(filelist) + FileA=ACS(i).Ain; + FileSTD=ACS(i).STD; + Aunique{i}=FileA(:,~ind_del); + STDunique{i}=FileSTD(~ind_del); +end +weightedA=ReducingA(Aunique,STDunique); + +%% 5 Determine Afinal that will be used to extract C's in each file. +Afinal=cat(2,commonA,weightedA); + +%%% Some processes making Afinal nicer, modified from Pengcheng Zhou's +%%% idea. +for i=1:size(Afinal,2) + ai=Afinal(:,i); + temp = full(ai>quantile(ai, 0.5, 1)); +% l = bwlabel(reshape(temp, File(1).options.d1, File(1).options.d2), 4); +% temp(l~=l(ind_ctr)) = false; + ai(~temp(:)) = 0; + Afinal(:,i)=ai; +end + +nz_ind=any(Afinal); +Afinal=Afinal(:,nz_ind); + +%% 6 "massive" procedure: Extract A from each file +filelist=dir(Datadir); +FILE(length(filelist)) = struct('A',[],'C',[],'ind_del',[]); +parfor i= 1:length(filelist) + picname=filelist(i).name(1:35) + nam=fullfile(datadir,filelist(i).name); + mode='massive'; + [~,FILE(i)]=demo_endoscope2(gSig,gSiz,min_pnr,bg_neuron_ratio,nam,mode,picname,Afinal,FILE(i)); +end +fprintf('Main extraction done'); + +neuron(length(filelist)) = struct('signal',[],'filelist',[]); +%%% Partition between those neurons found in each file and those not. +ind_del_final_cat=cat(2,FILE.ind_del); +ind_del_final=any(ind_del_final_cat,2); +parfor i= 1:length(filelist) + nA=FILE(i).A; + FILE(i).A=[nA(:,~ind_del_final) nA(:,ind_del_final)]; + fprintf('A extraction done'); + nC=FILE(i).C; + FILE(i).C=[nC(~ind_del_final,:);nC(ind_del_final,:)]; + fprintf('C extraction done'); + %%% save this data's origin + + for j=1:size(FILE(i).A,2) + jA=FILE(i).A(:,j); + jC=FILE(i).C(j,:); + neuron(i).signal(j,:)=median(jA(jA>0)*jC); + neuron(i).filelist=filelist(i); + end + fprintf('FILE %.0f extraction done\n', i); +end +fprintf('First %.0f neurons are found in each files while those after that are missing in some files', sum(ind_del_final)); +fprintf('ALL extraction done'); +eval(sprintf('save %sCNMFE_%s.mat %s -v7.3', outputdir, kind, 'neuron FILE')); + +%% 7 Post-analysis + diff --git a/BatchVerMO/demo_endoscope2.m b/BatchVerMO/demo_endoscope2.m new file mode 100644 index 0000000..be9c0e0 --- /dev/null +++ b/BatchVerMO/demo_endoscope2.m @@ -0,0 +1,251 @@ +function [A0s,File]=demo_endoscope2(bg_neuron_ratio,merge_thr,with_dendrites,K,start_frame,num_2read,name,neuron_full_partial,Mode,picname,File,Afinal,thresh_detecting_frames) +%function [A0s,File]=demo_endoscope2(gSig,gSiz,min_corr,min_pnr,min_pixel,bd,FS,SSub,TSub,bg_neuron_ratio,name,Mode,picname,Afinal,File,convolveType,merge_thr) +% [A0s,File]=demo_endoscope2(gSig,gSiz,min_corr,min_pnr,FS,SSub,TSub,bg_neuron_ratio,name,Mode,picname,Afinal,File) +% It is converted to function to cater to parfor loop. +% Meanwhile, some part of cnmf-e demo is applied to this sampling process while some simplied methods are +% applied for later for each file. Use 'mode' variable to specify this. +% In "initiation" mode, used in sampling data, most processes are the same as the original script, +% except that (1) initComponents_endoscope calls modified +% greedy_ROI_endoscope() which plots PNR and corr for seeds +% and non-neuron pixel. This will help you decide which PNR +% to use. +% (2) output is a File structure. File.options=neuron.options; File.Y=Y(raw signal); File.Ysignal=Ysignal; (Bg-subtracted and denoised Ysignal) +% and A0s, A0s is the output structure containing all A's obtained from each of the sample files. +% In "massive" mode, which is the process of applying the same A to all +% data in the folder, many steps are simplified. +% (1) No initiation. Just get C from A using background +% subtracted data. +% (2) Then Deconvolve C. +% In this mode, A0s is [], the File output only contains File.A=neuron.A; File.C=neuron.C; File.ind_del=ind_del; + +% modified by Shijie Gu from demo_endoscope script by Pengcheng Zhou. +%% set global variables +global d1 d2 numFrame ssub tsub sframe num2read Fs neuron neuron_ds ... + neuron_full Ybg_weights mode Picname nam outputdir; %#ok % global variables, don't change them manually +Picname=picname; +%% select data and map it to the RAM +mode=Mode; % 'initiation' mode or 'massive' mode +if strcmp(mode,'initiation') + nam=name; + cnmfe_choose_data; +elseif strcmp(mode,'massive') + nam=name{1}; + display(nam) + cnmfe_choose_data; + Ysignal=name{2}; +end +neuron_full=neuron_full_partial; +clear neuron_full_partial +neuron_full.updateParams('d1',d1, 'd2',d2); +min_pixel=neuron_full.options.min_pixel; +min_pnr=neuron_full.options.min_pnr; +min_corr=neuron_full.options.min_corr; + +%% create Source2D class object for storing results and parameters +Fs = neuron_full.Fs; % frame rate +ssub = neuron_full.options.ssub; % spatial downsampling factor +tsub = neuron_full.options.tsub; % temporal downsampling factor + +% with dendrites or not +if with_dendrites + % determine the search locations by dilating the current neuron shapes + neuron_full.options.search_method = 'dilate'; + neuron_full.options.bSiz = 20; +else + % determine the search locations by selecting a round area + neuron_full.options.search_method = 'ellipse'; + neuron_full.options.dist = 5; +end + + +%% downsample data for fast and better initialization +sframe=start_frame; % user input: first frame to read (optional, default:1) +if isempty(num_2read) + num2read = numFrame; % user input: how many frames to read (optional, default: until the end) +else + num2read = num_2read; +end + +tic; +cnmfe_load_data; +fprintf('Time cost in downsapling data: %.2f seconds\n', toc); + +Y = neuron.reshape(Y, 1); % convert a 3D video into a 2D matrix + +%% compute correlation image and peak-to-noise ratio image. +% cnmfe_show_corr_pnr; % this step is not necessary, but it can give you some... + % hints on parameter selection, e.g., min_corr & min_pnr + +%% initialization of A, C +% parameters +debug_on = false; % visualize the initialization procedue. +save_avi = false; %save the initialization procedure as an avi movie. +patch_par = [1,1]*1; %1; % divide the optical field into m X n patches and do initialization patch by patch. It can be used when the data is too large + +neuron.options.nk = 1; % number of knots for detrending + +% greedy method for initialization +tic; +if strcmp(mode,'initiation') + display(['working on ',nam]) + [center, Cn, pnr] = neuron.initComponents_endoscope(Y, K, patch_par, debug_on, save_avi); + fprintf('Time cost in initializing neurons: %.2f seconds\n', toc); + if isempty(neuron.A) + A0s=neuron.A; + File.options=[]; File.Ysignal=[]; File.neuron=[]; File.Ybg=[]; + clear global + return + end + % sort neurons + [~, srt] = sort(max(neuron.C, [], 2), 'descend'); + neuron.orderROIs(srt); + neuron_init = neuron.copy(); + +elseif strcmp(mode,'massive') + % parameters, estimate the background + spatial_ds_factor = 1; % spatial downsampling factor. it's for faster estimation + thresh = 10; % threshold for detecting frames with large cellular activity. (mean of neighbors' activity + thresh*sn) + [C,~]=extract_c(Ysignal,[],Afinal); + neuron.A=Afinal; + neuron.C_raw=C; + neuron.C=C; + [~,~]=neuron.updateTemporal_endoscope(Ysignal,false); + cnmfe_update_BG; + [~,~]=neuron.updateTemporal_endoscope(Ysignal,false); + A0s=[]; + File.neuron=neuron.copy(); + clear global + return +end + +%% iteratively update A, C and B +% parameters, merge neurons +display_merge = false; % visually check the merged neurons +view_neurons = false; % view all neurons + +% parameters, estimate the background +spatial_ds_factor = 1; % spatial downsampling factor. it's for faster estimation +thresh=thresh_detecting_frames; % threshold for detecting frames with large cellular activity. (mean of neighbors' activity + thresh*sn) +bg_neuron_ratio = bg_neuron_ratio; % spatial range / diameter of neurons + +% parameters, estimate the spatial components +update_spatial_method = 'hals'; % the method for updating spatial components {'hals', 'hals_thresh', 'nnls', 'lars'} +Nspatial = 5; % this variable has different meanings: + %1) udpate_spatial_method=='hals' or 'hals_thresh', + %then Nspatial is the maximum iteration + %2) update_spatial_method== 'nnls', it is the maximum + %number of neurons overlapping at one pixel + +% parameters for running iterations +nC = size(neuron.C, 1); % number of neurons + +maxIter = 2; % maximum number of iterations +miter = 1; +while miter <= maxIter + if strcmp(mode,'initiation') + %% merge neurons, order neurons and delete some low quality neurons + if miter ==1 + merge_thr = [1e-1, 0.8, .1]; % thresholds for merging neurons + % corresponding to {sptial overlaps, temporal correlation of C, + %temporal correlation of S} + else + merge_thr = merge_thr; + end + % merge neurons + cnmfe_quick_merge; % run neuron merges + if isempty(neuron.A); A0s=neuron.A; File.options=[]; File.Ysignal=[]; File.neuron=[]; File.Ybg=[]; + clear global; return; end + end + %% udpate background (cell 1, the following three blocks can be run iteratively) + % estimate the background + tic; + cnmfe_update_BG; + fprintf('Time cost in estimating the background: %.2f seconds\n', toc); + + %% update spatial & temporal components + tic; + for m=1:2 + %temporal + if strcmp(mode,'initiation') + neuron.updateTemporal_endoscope(Ysignal,true); + if isempty(neuron.A); + A0s=neuron.A; + clear global; return; end + cnmfe_quick_merge; % run neuron merges + if isempty(neuron.A); break; end + elseif strcmp(mode,'massive') + neuron.updateTemporal_endoscope(Ysignal,false); + end + %spatial + if strcmp(mode,'initiation') + neuron.updateSpatial_endoscope(Ysignal, Nspatial, update_spatial_method,[],true); + if isempty(neuron.A); break; end + neuron.trimSpatial(0.01, 3, min_pixel); % for each neuron, apply imopen first and then remove pixels that are not connected with the center + if isempty(neuron.A); break; end + if isempty(merged_ROI) + break; + end + elseif strcmp(mode,'massive') + neuron.updateSpatial_endoscope(Ysignal, Nspatial, update_spatial_method,[],false); + neuron.trimSpatial(0.01, 3, min_pixel, false); + end + end + fprintf('Time cost in updating spatial & temporal components: %.2f seconds\n', toc); + + %% pick neurons from the residual (cell 4). + if strcmp(mode,'initiation') + if miter==1 + neuron.options.seed_method = 'auto'; % methods for selecting seed pixels {'auto', 'manual'} + [center_new, Cn_res, pnr_res] = neuron.pickNeurons(Ysignal - neuron.A*neuron.C, patch_par); % method can be either 'auto' or 'manual' + end + if isempty(neuron.A); A0s=neuron.A; File.options=[]; File.Ysignal=[]; File.neuron=[]; File.Ybg=[]; + clear global; return; end + end + %% stop the iteration + temp = size(neuron.C, 1); + if or(nC==temp, miter==maxIter) + break; + else + miter = miter+1; + nC = temp; + end +end + +%if isempty(neuron.A); A0s=neuron.A; File.options=[]; File.Ysignal=[]; clear global; return; end +%Ybg=Ybg+b0; +%Ysignal_sn=Ysignal; +%noise=neuron.P.sn_neuron; +% if strcmp(mode,'initiation'); Ysignal=neuron.A*neuron.C_raw; end + +%% apply results to the full resolution +if or(ssub>1, tsub>1) + neuron_ds = neuron.copy(); % save the result + neuron = neuron_full.copy(); + cnmfe_full; + neuron_full = neuron.copy(); +end + +%% for 'massive' mode, save results +if strcmp(mode,'massive') + A0s=[]; + File.neuron=neuron.copy(); +end +%% for 'initiation' mode see and save results +resultstring=sprintf('%s_results', Picname); +neuron.viewNeurons([], neuron.C_raw, resultstring); +close(gcf); + +neuron.drawPNRCn(min_pnr,min_corr) +close(gcf); + +ColorAllNeurons(neuron.A,d1,d2,[Picname,strcat('PNR=',num2str(min_pnr)),'.png'],outputdir); +if strcmp(mode,'initiation') + A0s=neuron.A; + File.options=neuron.options; + File.neuron=neuron.copy(); + File.Ysignal=Ysignal; + File.Ybg=Ybg; +end +clear global +%globalVars = who('global'); +%eval(sprintf('save %s%s%s_results.mat %s', dir_nm, filesep, file_nm, strjoin(globalVars))); \ No newline at end of file diff --git a/BatchVerMO/drawgif.m b/BatchVerMO/drawgif.m new file mode 100644 index 0000000..f10ff02 --- /dev/null +++ b/BatchVerMO/drawgif.m @@ -0,0 +1,17 @@ +function drawgif(M3,d1,d2,filename,names) +if nargin<5 + names=1:numel(M3); +end +figure + for i = 1:numel(M3) + A=M3{i}; + ColorAllNeurons(A,d1,d2,['Day ' num2str(names(i))],[]); + h=getframe(1); + [A,map] = rgb2ind(h.cdata,256); + if i == 1 + imwrite(A,map,filename,'gif','LoopCount',Inf,'DelayTime',1); + else + imwrite(A,map,filename,'gif','WriteMode','append','DelayTime',1); + end + end +end \ No newline at end of file diff --git a/BatchVerMO/extract_a.m b/BatchVerMO/extract_a.m new file mode 100644 index 0000000..bcef299 --- /dev/null +++ b/BatchVerMO/extract_a.m @@ -0,0 +1,64 @@ +function [ai, ai_raw, ind_success] = extract_a(ci, Y_box, HY_box, Amask, ind_ctr, sz, sn, options) +% given the temporal component, background-subtracted and denoised calcium imaging data, extract +% spatial component of one neuron ai. Method is regression. + % Y=ac+Ybg+noise for non-denoised Y. + % Y=ac for denoised Y. + % Use sn==1 or [] to tell function which one suits current supply and demand. + % Input: + % for both, ind_ctr, sz, are needed for quality control. + %(1) Y=ac+Ybg+noise for non-denoised Y will need everything. + %(2) Y=ac for background subtracted Y and noise-including c. can have Amask=[] and Y=[]. +% Afterwards, +% we threshold the spatial shape and remove those too small or empty results. For those succeeded, then +% return an indicator ind_succes with value 1; otherwise, 0. +% some inputs explained: +% ind_ctr: scalar, location of the center +% sz: 2 X 1 vector, size of the patch + +%%Shijie Gu, techel@live.cn. modified from extract_ac() by Pengcheng Zhou. + +%% preparations +if isempty(ci) + error('No ci is provided.') +end +nr = sz(1); +nc = sz(2); +if isempty(options) + min_pixels=15; +else + min_pixels = options.min_pixels; +end + +HY=HY_box; + +%% estimate ai +if sn==1 + X=ci'; + temp = (X'*X)\(X'*HY'); + ai = max(0, temp'); +else + Y=Y_box; + T = length(ci); + y_bg = median(HY(~Amask, :), 1); + X = [ones(T,1), y_bg', ci']; + temp = (X'*X)\(X'*Y'); + ai = max(0, temp(3,:)'); +end + +%% +ai(isnan(ai))=0; +ai_raw=ai; +%% threshold the spatial shape and remove those too small or empty results. +% remove outliers not continuous with the main. +temp = full(ai>quantile(ai(:), 0.5)); +l = bwlabel(reshape(temp, nr, nc), 4); +temp(l~=l(ind_ctr)) = false; +ai(~temp(:)) = 0; + +if sum(ai(:)>0) < min_pixels %the ROI is too small + ind_success=false; +elseif norm(ai)==0 + ind_success= false; +else + ind_success=true; +end \ No newline at end of file diff --git a/BatchVerMO/extract_c.m b/BatchVerMO/extract_c.m new file mode 100644 index 0000000..22fe292 --- /dev/null +++ b/BatchVerMO/extract_c.m @@ -0,0 +1,42 @@ +function [ci,ind_success] = extract_c(Ysignal,Amask,A) +% [ci,ind_success] = extract_c(Ysignal,Amask,A) +% General description and Input: +% This function uses one of the two ways to extract temporal traces of a neuron. +% They all need one denoised/filtered/background-subtracted Ysiganl. +% When you only provide Amask of one neuron into the function, it uses this neuron's Amask to find in Ysignal all pixels involved in this neuron. +% The mean temperal trace of these pixels will become c. +% When you provide A, (it will ignore Amask), it will extract +% neuron's/neurons' temporal trace. The method is equation8 of CNMF-E +% conference script. +% Output: + % (1)Note that this function is on one-neuron scale for Amask mode. Use for loop outside to go + % through all of the neurons when you use it. If you provide A, + % this function can extract all the C's given all the A's. + % (2)No matter true or false is the "ind_success", ci will always have + % something. "ind_success" can be an indicator for further analysis decision + % such as, deconcolve or not, etc. +% This function is mainly used in A2C2A, but is also used by itself to + % extract c such as in the "massive" mode in "demo_endoscope2". + +%%Shijie Gu, techel@live.cn (ShanghaiTech University; Fee Lab at MIT-BCS) + +if ~isempty(A) + ind_nonzero = A>0; + ai_mask = mean(A(ind_nonzero))*ind_nonzero; + ci = (A-ai_mask)'*A\((A-ai_mask)'*Ysignal); + ci(isnan(ci)) = 0; +elseif ~isempty(Amask) + cs=Ysignal(Amask,:); + ci=mean(cs); + ci(isnan(ci)) = 0; +else + error('error in extract_c, please provide A or Amask.') +end + +if norm(ci)==0 % avoid empty results + ind_success=false; +else + ind_success=true; +end + +end \ No newline at end of file diff --git a/BatchVerMO/findn1n2.m b/BatchVerMO/findn1n2.m new file mode 100644 index 0000000..c4f614a --- /dev/null +++ b/BatchVerMO/findn1n2.m @@ -0,0 +1,150 @@ +function [ns_storage] = findn1n2(A1,A2,correlation_thresh,max2max2nd,skewnessthresh) +% [ns_storage] = findn1n2(A1,A2,correlation_thresh,max2max2nd,skewnessthresh) +% Some input explained: only those neurons that have coor coef with other neurons with skewness over +% skewnessthresh will be considered. See matlab function skewness(). +% The idea is that if this neuron is very independent, then its coor coef should only have very few big values. +% While those skewness with a lot of overlapping with other neurons +% will have many median level coor coef. skewness can very +% efficiently distinguish these. +% Output: if there is no tracked neurons over days. The output is []. +% This function is the base for TrackingNeuronOverDays.m and Over_Days_findAnn.m + +% by Shijie Gu, techel@live.cn, ShanghaiTech University, Harvard-MIT. + +if isempty(correlation_thresh) + correlation_thresh=0.8; +else + correlation_thresh=correlation_thresh; +end + +if isempty(max2max2nd) + max2max2nd=1.1; +else + max2max2nd=max2max2nd; +end + +if isempty(skewnessthresh) + skewnessthresh=5; +else + skewnessthresh=skewnessthresh; +end + + +%A1=neuron1.A; +A1=A1; +N1=1:size(A1,2); +%A2=neuron2.A; +A2=A2; +N2=1:size(A2,2); + +%% computing coor +% normalize + %CoorMatrix=(A1)'*(A2) + % A1norm=arrayfun(@(idx) norm(A1(:,idx)), 1:size(A1,2)); + % A2norm=arrayfun(@(idx) norm(A2(:,idx)), 1:size(A2,2)); + % CoefMatrix=bsxfun(@rdivide,CoorMatrix,A1norm'); + % CoefMatrix=bsxfun(@rdivide,CoefMatrix,A2norm); + +temp1 = bsxfun(@times, A1>0, 1./sqrt(sum(A1>0))); +temp2 = bsxfun(@times, A2>0, 1./sqrt(sum(A2>0))); +CoefMatrix = temp1'*temp2; + +%% filtering Coef +% (1) Each row/column must only have one value, its max. +% There might be situations where row/col has the same max value though +% rare. But this situation will be filtered out in the next step. +Filter1r=bsxfun(@eq,CoefMatrix,max(CoefMatrix,[],2)); +Filter1c=bsxfun(@eq,CoefMatrix,max(CoefMatrix,[],1)); +Filter1=and(Filter1r,Filter1c); + % The following is for the situation when there are two same max. + % Finding out rows with two maximums. +% [r,c]=find(Filter1); +% rc=[r,c]; +% [r_sorted,r_sort_index]=sort(r,'ascend'); +% rc_sorted=rc(r_sort_index,:); +% rowind=diff([0;r_sorted])==0; +% row_two_max=r_sorted(rowind); +% if length(row_two_max)>=1 +% protected=[]; +% protected2=[]; +% % (2) Assign one of the two max. +% for i=1:length(row_two_max) +% colind=find(rowind,i); +% colind=colind(end); +% col_ind=rc_sorted(colind-1,2); +% col_ind_2=rc_sorted(colind,2); +% if ~ismember(col_ind,protected) +% Filter1(row_two_max(i),col_ind_2)=0; +% protected=[protected col_ind] +% else +% if ~ismember(col_ind_2,protected2) +% Filter1(row_two_max(i),col_ind)=0; +% protected2=[protected2 col_ind_2] +% else +% Filter1(row_two_max(i),col_ind)=0; +% Filter1(row_two_max(i),col_ind_2)=0; +% end +% end +% end +% end + +% (2) Confidence that the neuron is the neuron we trace comes from the fact that +% (i) this correlation is unique. +% (ii) it is spatitally not overlapped with others. + +Coef_sorted=sort(CoefMatrix,2,'descend'); +try + ConfidentRow=(Coef_sorted(:,1)./Coef_sorted(:,2))>max2max2nd; + Filter3r=repmat(ConfidentRow,1,size(A2,2)); +catch + Filter3r=true(size(Coef_sorted,1),1); +end + +Coef_sorted2=sort(CoefMatrix,1,'descend'); +try + ConfidentCol=(Coef_sorted2(1,:)./Coef_sorted2(2,:))>max2max2nd; + Filter3c=repmat(ConfidentCol,size(A1,2),1); +catch + Filter3c=true(1,size(Coef_sorted,2)); +end +Filter3=Filter3r&Filter3c; + +%[skewness_sorted,ind]=sort(skewness(CoefMatrix,1,2),'descend'); +skew_ind=skewness(CoefMatrix,1,2); +ConfidentRow_skew=skew_ind>=skewnessthresh; +skew_ind_Col=skewness(CoefMatrix,1,1); +ConfidentCol_skew=skew_ind_Col>=skewnessthresh; +Filter4r=repmat(ConfidentRow_skew,1,size(A2,2)); +Filter4c=repmat(ConfidentCol_skew,size(A1,2),1); +Filter4=Filter4r&Filter4c; + +Filter2=CoefMatrix>correlation_thresh; + + +FilterCoef_temp=and(Filter1,Filter2); +FilterCoef_temp=and(FilterCoef_temp,Filter3); +FilterCoef=and(FilterCoef_temp,Filter4); + +% Calculating p-value +% t=CoefMatrix./sqrt(1-CoefMatrix.^2)*sqrt(size(A1,1)-2); +%(http://support.minitab.com/en-us/minitab-express/1/help-and-how-to/modeling-statistics/regression/how-to/correlation/methods-and-formulas/) + + % Let n be your sample size (size(A1,1)) + % Let v be your degrees of freedom (2) + % Then: (https://stackoverflow.com/questions/10617050/how-to-calculate-p-value-for-t-test-in-matlab) +% pvalues = 2*(1-tcdf(abs(t),size(A1,1)-2)); +% Filterpvalue = pvalues<=0.5; +%% +% Filter= and(Filterpvalue,FilterCoef); +Filter=FilterCoef; +corrnumber=sum(sum(Filter,1),2); +fprintf('There are %.0f pairs of neurons tracked over two days.\n', corrnumber); + +[n1,n2] = find(Filter); +ns_storage=[n1,n2]; +% n1_the_rest=setdiff(N1,n1);% returns the data in N1 that is not in n1 +% n2_the_rest=setdiff(N2,n2); +% +% neuron1.APermuted=A1(:,[n1' n1_the_rest]); +% neuron2.APermuted=A2(:,[n2' n2_the_rest]); +end \ No newline at end of file diff --git a/BatchVerMO/greedyROI_endoscope.asv b/BatchVerMO/greedyROI_endoscope.asv new file mode 100644 index 0000000..468ab23 --- /dev/null +++ b/BatchVerMO/greedyROI_endoscope.asv @@ -0,0 +1,452 @@ +function [results, center, Cn, PNR, save_avi,HY0,ind_localmax1] = greedyROI_endoscope(Y, K, options,debug_on, save_avi) +% (parameter sweep version, modified by Shijie Gu) +% A greedy method for detecting ROIs and initializing CNMF. In each iteration, +% it searches the one with large (peak-median)/noise level and large local +% correlation. It's the same with greedyROI_corr.m, but with some features +% specialized for endoscope data. +% This function has a plotting section (interwoven with the original code) added that shows you seeds' pnr&corr +% as well as that of pixels not included in any neurons. In the same plot also +% shows you the min_pnr*min_corr curve. This helps you +% investigate your choice of PNR or corr on the amount and quality of +% neurons picked. The actual plotting will be done after iteration +% where neuron deletion and additions are possible. Method +% drawPNRCn() is called in demo_endoscope2 to generate the plot. +%% Input: +% Y: d X T matrx, imaging data +% K: scalar, maximum number of neurons to be detected. +% options: struct data of paramters/options +% d1: number of rows +% d2: number of columns +% gSiz: maximum size of a neuron +% nb: number of background +% min_corr: minimum threshold of correlation for segementing neurons +% sn: d X 1 vector, noise level of each pixel +% debug_on: options for showing procedure of detecting neurons +% save_avi: save the video of initialization procedure. string: save +% video; true: just play it; false: interactive mode. (the name of this +% argument is very misleading after several updates of the code. sorry) +%% Output: +%` results: struct variable with fields {'Ain', 'Cin', 'Sin', 'kernel_pars'} +% Ain: d X K' matrix, estimated spatial component +% Cin: K'X T matrix, estimated temporal component +% Sin: K' X T matrix, inferred spike counts within each frame +% kernel_pars: K'X1 cell, parameters for the convolution kernel +% of each neuron +% THRESH: structure, 4 fields, Corr, correlation value (intact orignal Cn) of pixels of neuron seeds. +% CorrOut, correlation value (intact orignal Cn) of pixels that are not involved in any neuron shapes. +% similarly for PNR, and PNROut. +% This will go into neuron.P.THRESH eventually (initComponents_endoscope will do this.). +% center: K' X 2, coordinate of each neuron's center +% Cn: d1*d2, correlation image +% save_avi: options for saving avi. + +%% Author: Pengcheng Zhou, Carnegie Mellon University. zhoupc1988@gmail.com +% the method is an modification of greedyROI method used in Neuron paper of Eftychios +% Pnevmatikakis et.al. https://github.com/epnev/ca_source_extraction/blob/master/utilities/greedyROI2d.m +% In each iteration of peeling off neurons, it searchs the one with maximum +% value of (max-median)/noise * Cn, which achieves a balance of SNR and +% local correlation. + + +%% use correlation to initialize NMF +%% parameters + +d1 = options.d1; % image height +d2 = options.d2; % image width +gSig = options.gSig; % width of the gaussian kernel approximating one neuron +gSiz = options.gSiz; % average size of neurons +min_corr = options.min_corr; %minimum local correlations for determining seed pixels +min_pnr = options.min_pnr; % peak to noise ratio for determining seed pixels +min_v_search = min_corr*min_pnr; +seed_method = options.seed_method; % methods for selecting seed pixels +% kernel_0 = options.kernel; +deconv_options_0= options.deconv_options; +min_pixel = options.min_pixel; % minimum number of pixels to be a neuron +deconv_flag = options.deconv_flag; +% smin = options.smin; +% boudnary to avoid for detecting seed pixels +THRESH=struct('Corr',[],'PNR',[],'CorrOut',[],'PNROut',[],'pixelr',[],'pixelc',[]); +try + bd = options.bd; +catch + bd = round(gSiz/2); +end +sig = 5; % thresholding noise by sig*std() + +% exporting initialization procedures as a video +if ~exist('save_avi', 'var')||isempty(save_avi) + save_avi = false; +elseif ischar(save_avi) + avi_name = save_avi; + debug_on = true; +elseif save_avi + debug_on = true; % turn on debug mode +else + save_avi = false; %don't save initialization procedure +end +% debug mode and exporting results +if ~exist('debug_on', 'var') + debug_on = false; +end + +if ~ismatrix(Y); Y = reshape(Y, d1*d2, []); end; % convert the 3D movie to a matrix +Y(isnan(Y)) = 0; % remove nan values +Y = double(Y); +T = size(Y, 2); + +%% preprocessing data +% create a spatial filter for removing background +psf = fspecial('gaussian', round(gSiz), gSig); +if options.center_psf + ind_nonzero = (psf(:)>=max(psf(:,1))); + psf = psf-mean(psf(ind_nonzero)); + psf(~ind_nonzero) = 0; +end + + +% filter the data +HY = imfilter(reshape(Y, d1,d2,[]), psf, 'replicate'); +HY = reshape(HY, d1*d2, []); +% HY_med = median(HY, 2); +% HY_max = max(HY, [], 2)-HY_med; % maximum projection +HY0= HY; +HY = bsxfun(@minus, HY, median(HY, 2)); + + +HY_max = max(HY, [], 2); +Ysig = get_noise_fft(HY, options); +PNR = reshape(HY_max./Ysig, d1, d2); +PNR0 = PNR; +%PNR(PNR + set(gcf, 'defaultAxesFontSize', 20); + ax_cn = axes('position', [0.04, 0.5, 0.3, 0.4]); + ax_pnr_cn = axes('position', [0.36, 0.5, 0.3, 0.4]); + ax_cn_box = axes('position', [0.68, 0.54, 0.24, 0.32]); + ax_trace = axes('position', [0.05, 0.05, 0.92, 0.4]); + axes(ax_cn); + imagesc(Cn0); + % imagesc(Cn.*PNR, quantile(Cn(:).*PNR(:), [0.5, 0.99])); + axis equal off tight; hold on; + % title('Cn * PNR'); + title('Cn'); + if exist('avi_name', 'var') + avi_file = VideoWriter(avi_name); + avi_file.FrameRate = 1; + avi_file.open(); + elseif save_avi + avi_file = VideoWriter('temp.avi'); + avi_file.FrameRate = 1; + avi_file.open(); + end + +end + +%% start initialization +if ~exist('K', 'var')||isempty(K); + K = floor(sum(v_search(:)>0)/10); +else + K = min(floor(sum(v_search(:)>0)/10), K); +end +Ain = zeros(d1*d2, K); % spatial components +Cin = zeros(K, T); % temporal components +Sin = zeros(K, T); % spike counts +Cin_raw = zeros(K, T); +kernel_pars = cell(K,1); % parameters for the convolution kernels of all neurons +center = zeros(K, 2); % center of the initialized components + +%% do initialization in a greedy way +searching_flag = true; +k = 0; %number of found components +% set boundary to be 0 +ind_bd = false(size(v_search)); +ind_bd(1:bd, :) = true; +ind_bd((end-bd+1):end, :) = true; +ind_bd(:, 1:bd) = true; +ind_bd(:, (end-bd):end) = true; +while searching_flag + %% find local maximum as initialization point + %find all local maximum as initialization point + tmp_d = 2*round(gSiz/4)+1; + v_search = medfilt2(v_search, round(gSiz/4)*[1, 1]); %+randn(size(v_search))*(1e-100); + v_search(ind_search) = 0; + v_max = ordfilt2(v_search, tmp_d^2, true(tmp_d)); + % set boundary to be 0 + v_search(ind_bd) = 0; + + if strcmpi(seed_method, 'manual') %manually select seed pixels + tmp_fig = figure('position', [200, 200, 1024, 412]); + subplot(121); cla; + imagesc(Cn0.*PNR0); hold on; + title('Cn*PNR'); + plot(center(1:k, 2), center(1:k, 1), '*r'); + axis equal off tight; + subplot(122); + imagesc(v_search.*Cn0.*PNR0); %, [0, max(max(min_v_search(:)*0.99), min_v_search)]); + hold on; + axis equal tight; drawnow; + set(gca, 'xtick', []); + set(gca, 'ytick', []); + title('click neuron centers for initialziation'); + xlabel('click invalid pixels to stop', 'color', 'r'); + ind_localmax = zeros(K,1); + for tmp_k=1:K + figure(tmp_fig); + [tmp_x, tmp_y] = ginput(1); + tmp_x = round(tmp_x); tmp_y = round(tmp_y); + if isempty(tmp_x)||or(tmp_x<1, tmp_x>d2) || or(tmp_y<1, tmp_y>d1) ||(v_search(tmp_y, tmp_x)==0) + break; + end + plot(tmp_x, tmp_y, '*r', 'linewidth', 2); + drawnow(); + ind_localmax(tmp_k) = sub2ind([d1,d2], tmp_y, tmp_x); + end + close(tmp_fig); + + ind_localmax = ind_localmax(1:(tmp_k-1)); + if isempty(ind_localmax) + break; + end + else + % automatically select seed pixels + ind_search(v_search0)); + + if(isempty(ind_localmax)); break; end + end + [~, ind_sort] = sort(v_search(ind_localmax), 'descend'); + ind_localmax = ind_localmax(ind_sort); + [r_peak, c_peak] = ind2sub([d1,d2],ind_localmax); + + %% try initialization over all local maximums + for mcell = 1:length(ind_localmax); + % find the starting point + ind_p = ind_localmax(mcell); + % max_v = max_vs(mcell); + max_v = v_search(ind_p); + ind_search(ind_p) = true; % indicating that this pixel has been searched. + if max_v=1) && any(corr(Cin(1:k, :)', y0')>0.9) %already found similar temporal traces +% continue; +% end + if max(diff(y0))< 3*y0_std % signal is weak + continue; + end + + % select its neighbours for estimation of ai and ci, the box size is + %[2*gSiz+1, 2*gSiz+1] + rsub = max(1, -gSiz+r):min(d1, gSiz+r); + csub = max(1, -gSiz+c):min(d2, gSiz+c); + [cind, rind] = meshgrid(csub, rsub); + [nr, nc] = size(cind); + ind_nhood = sub2ind([d1, d2], rind(:), cind(:)); + HY_box = HY(ind_nhood, :); % extract temporal component from HY_box + Y_box = Y(ind_nhood, :); % extract spatial component from Y_box + ind_ctr = sub2ind([nr, nc], r-rsub(1)+1, c-csub(1)+1); % subscripts of the center + + % neighbouring pixels to update after initialization of one + % neuron + rsub = max(1, -2*gSiz+r):min(d1, 2*gSiz+r); + csub = max(1, -2*gSiz+c):min(d2, 2*gSiz+c); + [cind, rind] = meshgrid(csub, rsub); + ind_nhood_HY = sub2ind([d1, d2], rind(:), cind(:)); + [nr2, nc2] = size(cind); + + %% show temporal trace in the center + if debug_on + axes(ax_pnr_cn); cla; + imagesc(reshape(v_search, d1, d2), [0, max_v]); + title(sprintf('neuron %d', k+1)); + axis equal off tight; hold on; + plot(c_peak(mcell:end), r_peak(mcell:end), '.r'); + plot(c,r, 'or', 'markerfacecolor', 'r', 'markersize', 10); + axes(ax_cn_box); + imagesc(reshape(Cn(ind_nhood), nr, nc), [0, 1]); + axis equal off tight; + title('correlation image'); + axes(ax_trace); cla; + plot(HY_box(ind_ctr, :)); title('activity in the center'); axis tight; + if ~save_avi; pause; end + if exist('avi_file', 'var') + frame = getframe(gcf); + frame.cdata = imresize(frame.cdata, [800, 1200]); + avi_file.writeVideo(frame); + end + end + + %% extract ai, ci + sz = [nr, nc]; + if options.center_psf + [ai, ci_raw, ind_success] = extract_ac(HY_box, Y_box, ind_ctr, sz); + else + [ai, ci_raw, ind_success] = extract_ac(HY_box, Y_box, ind_ctr, sz); + if options.gaussian_shape && ind_success + ai = spatial_constraints(reshape(ai, sz)); + ai = ai(:); + end + end + + if or(any(isnan(ai)), any(isnan(ci_raw))); ind_success=false; end + % if max(ci_raw)0; + ind_neuron(ind_p)=true; + THRESH.PNR=[THRESH.PNR PNR0(ind_p)]; + THRESH.Corr=[THRESH.Corr Cn0(ind_p)]; + THRESH.pixelr=[THRESH.pixelr r]; + THRESH.pixelc=[THRESH.pixelc c]; + + % avoid searching nearby pixels that are highly correlated with this one + ind_search(ind_nhood(ai>max(ai)*options.merge_thr)) = true; + + % update the raw data + Y(ind_nhood, :) = Y_box - ai*ci_raw; + % update filtered data + Hai = imfilter(reshape(Ain(ind_nhood_HY, k), nr2, nc2), psf, 'replicate'); + HY_box = HY(ind_nhood_HY, :) - Hai(:)*ci_raw; + % HY_box = bsxfun(@minus, HY_box, median(HY_box, 2)); + HY(ind_nhood_HY, :) = HY_box; + + % update the maximum projection of HY + Ysig_box = Ysig(ind_nhood_HY); + temp = max(HY_box, [], 2); + tmp_PNR = temp./Ysig_box; + tmp_PNR(or(isnan(tmp_PNR), tmp_PNR=max(psf(:,1))); + psf = psf-mean(psf(ind_nonzero)); + psf(~ind_nonzero) = 0; +end + + +% filter the data +HY = imfilter(reshape(Y, d1,d2,[]), psf, 'replicate'); +HY = reshape(HY, d1*d2, []); +% HY_med = median(HY, 2); +% HY_max = max(HY, [], 2)-HY_med; % maximum projection +HY = bsxfun(@minus, HY, median(HY, 2)); +HY0= HY; + +HY_sorted = sort(HY,2,'descend'); +HY_max = mean(HY_sorted(:,1:3),2); % choose top 3 max to avoid noisy fluctuation. +Ysig = get_noise_fft(HY, options); +PNR = reshape(HY_max./Ysig, d1, d2); +PNR0 = PNR; +%PNR(PNR0)); + min_pnr=threshold_by_diff(PNR0(ind_localmax0),1,10); + results.min_pnr=min_pnr; % min_pnr will be added to neuron.options outside this function through 'results'. + min_v_search = min_corr*min_pnr; + +v_search(or(Cn + set(gcf, 'defaultAxesFontSize', 20); + ax_cn = axes('position', [0.04, 0.5, 0.3, 0.4]); + ax_pnr_cn = axes('position', [0.36, 0.5, 0.3, 0.4]); + ax_cn_box = axes('position', [0.68, 0.54, 0.24, 0.32]); + ax_trace = axes('position', [0.05, 0.05, 0.92, 0.4]); + axes(ax_cn); + imagesc(Cn0); + % imagesc(Cn.*PNR, quantile(Cn(:).*PNR(:), [0.5, 0.99])); + axis equal off tight; hold on; + % title('Cn * PNR'); + title('Cn'); + if exist('avi_name', 'var') + avi_file = VideoWriter(avi_name); + avi_file.FrameRate = 1; + avi_file.open(); + elseif save_avi + avi_file = VideoWriter('temp.avi'); + avi_file.FrameRate = 1; + avi_file.open(); + end + +end + +%% start initialization +if ~exist('K', 'var')||isempty(K); + K = floor(sum(v_search(:)>0)/10); % I strongly suggest replace 10 with gSiz or gSig. +else + K = min(floor(sum(v_search(:)>0)/10), K); +end +Ain = zeros(d1*d2, K); % spatial components +Cin = zeros(K, T); % temporal components +Sin = zeros(K, T); % spike counts +Cin_raw = zeros(K, T); +kernel_pars = cell(K,1); % parameters for the convolution kernels of all neurons +center = zeros(K, 2); % center of the initialized components + +%% do initialization in a greedy way +searching_flag = true; +k = 0; %number of found components +% set boundary to be 0 +ind_bd = false(size(v_search)); +ind_bd(1:bd, :) = true; +ind_bd((end-bd+1):end, :) = true; +ind_bd(:, 1:bd) = true; +ind_bd(:, (end-bd):end) = true; + +while searching_flag + %% find local maximum as initialization point + %find all local maximum as initialization point + tmp_d = 2*round(gSiz/4)+1; + v_search = medfilt2(v_search, round(gSiz/4)*[1, 1]); %+randn(size(v_search))*(1e-100); + v_search(ind_search) = 0; + v_max = ordfilt2(v_search, tmp_d^2, true(tmp_d)); + % set boundary to be 0 + v_search(ind_bd) = 0; + v_search0=v_search; + + if strcmpi(seed_method, 'manual') %manually select seed pixels + tmp_fig = figure('position', [200, 200, 1024, 412]); + subplot(121); cla; + imagesc(Cn0.*PNR0); hold on; + title('Cn*PNR'); + plot(center(1:k, 2), center(1:k, 1), '*r'); + axis equal off tight; + subplot(122); + imagesc(v_search.*Cn0.*PNR0); %, [0, max(max(min_v_search(:)*0.99), min_v_search)]); + hold on; + axis equal tight; drawnow; + set(gca, 'xtick', []); + set(gca, 'ytick', []); + title('click neuron centers for initialziation'); + xlabel('click invalid pixels to stop', 'color', 'r'); + ind_localmax = zeros(K,1); + for tmp_k=1:K + figure(tmp_fig); + [tmp_x, tmp_y] = ginput(1); + tmp_x = round(tmp_x); tmp_y = round(tmp_y); + if isempty(tmp_x)||or(tmp_x<1, tmp_x>d2) || or(tmp_y<1, tmp_y>d1) ||(v_search(tmp_y, tmp_x)==0) + break; + end + plot(tmp_x, tmp_y, '*r', 'linewidth', 2); + drawnow(); + ind_localmax(tmp_k) = sub2ind([d1,d2], tmp_y, tmp_x); + end + close(tmp_fig); + + ind_localmax = ind_localmax(1:(tmp_k-1)); + if isempty(ind_localmax) + break; + end + else + % automatically select seed pixels + ind_search(v_search0)); + %display(ind_localmax) + if(isempty(ind_localmax)); break; end + end + [~, ind_sort] = sort(v_search(ind_localmax), 'descend'); + ind_localmax = ind_localmax(ind_sort); + [r_peak, c_peak] = ind2sub([d1,d2],ind_localmax); + + %% try initialization over all local maximums + for mcell = 1:length(ind_localmax); + % find the starting point + ind_p = ind_localmax(mcell); + % max_v = max_vs(mcell); + max_v = v_search(ind_p); + ind_search(ind_p) = true; % indicating that this pixel has been searched. + if max_v=1) && any(corr(Cin(1:k, :)', y0')>0.9) %already found similar temporal traces +% continue; +% end + if max(diff(y0))< 3*y0_std % signal is weak + continue; + end + + % select its neighbours for estimation of ai and ci, the box size is + %[2*gSiz+1, 2*gSiz+1] + rsub = max(1, -gSiz+r):min(d1, gSiz+r); + csub = max(1, -gSiz+c):min(d2, gSiz+c); + [cind, rind] = meshgrid(csub, rsub); + [nr, nc] = size(cind); + ind_nhood = sub2ind([d1, d2], rind(:), cind(:)); + HY_box = HY(ind_nhood, :); % extract temporal component from HY_box + Y_box = Y(ind_nhood, :); % extract spatial component from Y_box + ind_ctr = sub2ind([nr, nc], r-rsub(1)+1, c-csub(1)+1); % subscripts of the center + + % neighbouring pixels to update after initialization of one + % neuron + rsub = max(1, -2*gSiz+r):min(d1, 2*gSiz+r); + csub = max(1, -2*gSiz+c):min(d2, 2*gSiz+c); + [cind, rind] = meshgrid(csub, rsub); + ind_nhood_HY = sub2ind([d1, d2], rind(:), cind(:)); + [nr2, nc2] = size(cind); + + %% show temporal trace in the center + if debug_on + axes(ax_pnr_cn); cla; + imagesc(reshape(v_search, d1, d2), [0, max_v]); + title(sprintf('neuron %d', k+1)); + axis equal off tight; hold on; + plot(c_peak(mcell:end), r_peak(mcell:end), '.r'); + plot(c,r, 'or', 'markerfacecolor', 'r', 'markersize', 10); + axes(ax_cn_box); + imagesc(reshape(Cn(ind_nhood), nr, nc), [0, 1]); + axis equal off tight; + title('correlation image'); + axes(ax_trace); cla; + plot(HY_box(ind_ctr, :)); title('activity in the center'); axis tight; + if ~save_avi; pause; end + if exist('avi_file', 'var') + frame = getframe(gcf); + frame.cdata = imresize(frame.cdata, [800, 1200]); + avi_file.writeVideo(frame); + end + pause(3) + end + + %% extract ai, ci + sz = [nr, nc]; + if options.center_psf + [ai, ci_raw, ind_success] = extract_ac(HY_box, Y_box, ind_ctr, sz); + else + [ai, ci_raw, ind_success] = extract_ac(HY_box, Y_box, ind_ctr, sz); + if options.gaussian_shape && ind_success + ai = spatial_constraints(reshape(ai, sz)); + ai = ai(:); + end + end + + if or(any(isnan(ai)), any(isnan(ci_raw))); ind_success=false; end + % if max(ci_raw)0; + ind_neuron(ind_p)=true; + THRESH.PNR=[THRESH.PNR PNR0(ind_p)]; + THRESH.Corr=[THRESH.Corr Cn0(ind_p)]; + + % avoid searching nearby pixels that are highly correlated with this one + ind_search(ind_nhood(ai>max(ai)*options.merge_thr)) = true; + + % update the raw data + Y(ind_nhood, :) = Y_box - ai*ci_raw; + % update filtered data + Hai = imfilter(reshape(Ain(ind_nhood_HY, k), nr2, nc2), psf, 'replicate'); + HY_box = HY(ind_nhood_HY, :) - Hai(:)*ci_raw; + % HY_box = bsxfun(@minus, HY_box, median(HY_box, 2)); + HY(ind_nhood_HY, :) = HY_box; + + % update the maximum projection of HY + Ysig_box = Ysig(ind_nhood_HY); + temp_maxes = sort(HY_box, 2, 'descend'); + temp=mean(temp_maxes(:,1:3),2); + tmp_PNR = temp./Ysig_box; + tmp_PNR(or(isnan(tmp_PNR), tmp_PNR 3*STD as +% criteria for not being noise, then, merge neurons based on +% spatial, temporal correlation, and their distance. +% input: +% A: concatnated A from neurons from one or many files. +% ACS: structure, having fields of Ain, Cin (concatnated A and C from +% neurons from all files), and std of Cin(STD). +% merge_thr: 1X2 vector, threshold for two metrics {'A', 'C'}. it merge neurons based +% on correlations of spatial shapes ('A'), calcium traces ('C'). +% dmin: min distance for two neurons to be called different. +% d1 and d2: row and column of the FOV, for calculating center of each +% neuron. +% output: +% Afinal: merged As. As are merged using sum of weighted As, weight is max(diff(C,1,2))./STD; +% newIDs: cell array, dim: 1*(number of neurons after merging). Each +% cell element has the orginal neuron number it has merged from (the +% nueron number is cumsum across the second dim of A0s). +% close_ind: ind of neurons for output (that are merged) just because some neurons are +% close together. +% real_ind: neuron ind for input A's column as real neurons. +% Other outputs are the same as the original quickMerge(). +% merged_ROIs + +% Author: Shijie Gu, techel@live.cn, modified from quickMerge() by Pengcheng Zhou +% The basic idea is proposed by Eftychios A. Pnevmatikakis: high temporal correlation + spatial overlap +% reference: Pnevmatikakis et.al.(2016). Simultaneous Denoising, Deconvolution, and Demixing of Calcium Imaging Data. Neuron +%% variables & parameters +if ~exist('merge_thr', 'var') || isempty(merge_thr) || numel(merge_thr)~=2 + merge_thr = [0.7, 0.7]; +end + +A_thr = merge_thr(1); +C_thr = merge_thr(2); + +C=cat(2,ACS.Cin); +STD=std(C,1,2); + +real_ind=max(diff(C,1,2),[],2)> 3*STD; %not noise +A=A(:,real_ind); +C=C(real_ind,:); +STD=STD(real_ind); +STD=max(diff(C,1,2))./STD; + +Amask=A>0; +K = size(C,1); % number of neurons +%% find neuron pairs to merge +% compute spatial correlation +temp = bsxfun(@times, Amask, 1./sqrt(sum(Amask))); +clear Amask +A_overlap = temp'*temp; + +% compute temporal correlation +C_corr = corr(C')-eye(K); + +% compute center distance +ctr = round(com(A, d1, d2)); +yy = ctr(:,1); +xx = ctr(:,2); +dist_v = sqrt(bsxfun(@minus, xx, xx').^2 + bsxfun(@minus, yy, yy').^2); + + +% using merging criterion to detect paired neurons +flag_merge1 = (A_overlap>A_thr)&(C_corr>C_thr); +flag_merge2 = (dist_v<=dmin); +flag_merge=or(flag_merge1,flag_merge2); + +MC=merge_detail(flag_merge); +MC1=merge_detail(flag_merge1); +[~,close_ind]=setdiff(MC',MC1','rows','stable'); + +%%%% +% [l,c] = graph_connected_comp(sparse(flag_merge)); % extract connected components +% MC = bsxfun(@eq, reshape(l, [],1), 1:c); +% MC(:, sum(MC,1)==1) = []; +%%%% + +if isempty(MC) + fprintf('All pairs of neurons are below the merging criterion.\n\n'); + newIDs=[]; merged_ROIs=[]; + return; +else + fprintf('%d neurons will be merged into %d new neurons\n\n', sum(MC(:)), size(MC,2)); +end + +[nr, n2merge] = size(MC); +merged_ROIs = cell(n2merge,1); +newIDs=cell(1,nr); %newIDs = num2cell(1:nr); +ind_del=true(nr,1); +Afinal=zeros(size(A)); + + +% start merging +for m=1:n2merge + IDs = find(MC(:, m)); % IDs of neurons within this cluster + merged_ROIs{m} = IDs; + + % determine searching area + active_pixel = sum(A(:,IDs), 2)>0; + + + % update spatial/temporal components of the merged neuron + % data = A(active_pixel, IDs)*C(IDs, :); +%%%%%%%%% Older version uses this method +% data=[]; +% for i=1:numel(ACS) +% FileC=ACS(i).Cin; +% data = [data A(active_pixel, IDs)*FileC(IDs, :)]; +% end +% +% data=data./length(IDs); +% [~,I] = max(std(C(IDs, :),0,2)); % choose the most confident(with biggest std) ci. +% ci=C(IDs(I),:); +% for miter=1:10 +% ai = data*ci'/(ci*ci'); +% ci = ai'*data/(ai'*ai); +% end +%%%%%%%% + A_temp=A(active_pixel,MC(:,m)); + STD_temp=STD(MC(:,m)); + catSTD=STD_temp./sum(STD_temp); + + weightedA=A_temp*catSTD; + + ind_del(IDs(1))= false; + newIDs{IDs(1)} = IDs; + Afinal(active_pixel,IDs(1))=weightedA; + +end + + +newIDs(ind_del) = []; +Afinal(:,ind_del) = []; + +end diff --git a/BatchVerMO/mergeACforMo.m b/BatchVerMO/mergeACforMo.m new file mode 100644 index 0000000..f1f6fe1 --- /dev/null +++ b/BatchVerMO/mergeACforMo.m @@ -0,0 +1,147 @@ +function [Afinal_alldays,MC,newIDs,merged_ROIs,close_ind,eachday_ind] = mergeACforMo(A,C,merge_thr,M,dmin,d1,d2) +% General Description: Merge neurons based on +% spatial, temporal correlation, and their distance. Afinal will be +% different for each day so that they are put into cell array where each day, +% has its own cell. Yet, each day's A is merged from the exact neurons using the +% exact weight matrix: max(diff(C,1,2))./STD, see 'C' in input. This +% function is key in tracking neuron. +% This function is very similar to mergeAC, but this one does not +% eliminate neurons with low signal. It also requires C as opposed to +% ACS. Its emphasis is on the handling of multi-day data in the same way. +% input: +% A: concatenated A from neurons from many files registered towards +% one day. That day should be the most confidently registered A. +% C: BigC(in ReadMe) of each file concatenated in time. +% merge_thr: 1X2 vector, threshold for two metrics {'A', 'C'}. it merge neurons based +% on correlations of spatial shapes ('A'), calcium traces ('C'). +% M: cell array of length number of days. Within each day's cell, it +% is As registered towards that day. More detailed in ReadMe for MoBatchVer. +% dmin: min distance for two neurons to be called different. +% d1 and d2: row and column of the FOV, for calculating center of each +% neuron. +% output: +% Afinal_alldays: merged As for each day. Each day's cell array is put +% into a seperate cell. For the following output variables, it is essentially +% the same as MergeAC. For ForMo version, each result becomes a cell +% array, with ex +% MC: see 'help merge_detail'. +% newIDs: cell array, dim: 1*(number of neurons after merging). Each +% cell element has the orginal neuron number it has merged from (the +% nueron number is cumsum across the second dim of A0s). +% merged_ROIs: essentially the same as newIDs. Left over from previous +% version. Keep if for now. +% close_ind: ind of neurons for output (that are merged) just because some neurons are +% close together. +% eachday_ind: numbers of neurons that show up in each day. + +% Author: Shijie Gu, techel@live.cn, modified from quickMerge() by Pengcheng Zhou +% The basic idea is proposed by Eftychios A. Pnevmatikakis: high temporal +% correlation + spatial overlap +% reference: Pnevmatikakis et.al.(2016). Simultaneous Denoising, Deconvolution, and Demixing of Calcium Imaging Data. Neuron +%% variables & parameters +if ~exist('merge_thr', 'var') || isempty(merge_thr) || numel(merge_thr)~=2 + merge_thr = [0.7, 0.7]; +end + +A_thr = merge_thr(1); +C_thr = merge_thr(2); + +STD=std(C,1,2); +STD=max(diff(C,1,2))./STD; + +K = size(C,1); % number of neurons +%% find neuron pairs to merge +% compute spatial correlation +Amask=A>0; +temp = bsxfun(@times, Amask, 1./sqrt(sum(Amask))); +clear Amask +A_overlap = temp'*temp; + +% compute temporal correlation +C_corr = corr(C')-eye(K); + +% compute center distance +ctr = round(com(A, d1, d2)); +yy = ctr(:,1); +xx = ctr(:,2); +dist_v = sqrt(bsxfun(@minus, xx, xx').^2 + bsxfun(@minus, yy, yy').^2); + +% using merging criterion to detect paired neurons +flag_merge1 = (A_overlap>A_thr)&(C_corr>C_thr); +flag_merge2 = (dist_v<=dmin); +flag_merge=or(flag_merge1,flag_merge2); + +MC=merge_detail(flag_merge); +MC_=MC(:,sum(MC,1)>=numel(M));% rough sift +[MC_,~]=merge_with_eachday(M,MC_); % detailed sift +eachday_ind=size(MC_,2); + +MC1=merge_detail(flag_merge1); +MC1_=MC1(:,sum(MC1,1)>=numel(M)); +[MC1_,~]=merge_with_eachday(M,MC1_); + + +[MC_missing,missingday_ind]=setdiff(MC',MC_','rows','stable'); +[~,close_ind]=setdiff(MC',MC1','rows','stable'); +MC=[MC_,MC_missing']; + +clear A_overlap C_corr C flag_merge1 flag_merge2 flag_merge; +display('Deleted some big variables.') + + +merge_Bool=sum(MC,2)>0; +Aunique_Bool=~merge_Bool; %keep it here for now, future versions may need this. + +%%%% +% [l,c] = graph_connected_comp(sparse(flag_merge)); % extract connected components +% MC = bsxfun(@eq, reshape(l, [],1), 1:c); +% MC(:, sum(MC,1)==1) = []; +%%%% +if isempty(MC) + fprintf('All pairs of neurons are below the merging criterion!\n\n'); + newIDs=[]; merged_ROIs=[]; + return; +else + fprintf('%d neurons will be merged into %d new neurons\n\n', sum(MC(:)), size(MC,2)); +end + +[nr, n2merge] = size(MC); +%merged_ROIs_alldays=cell(1,numel(M)); +%newIDs_alldays=cell(1,numel(M)); +Afinal_alldays=cell(1,numel(M)); + +for i=1:numel(M) + Afinal=zeros(size(A)); + ind_del=true(nr,1); + if i==1 + merged_ROIs = cell(n2merge,1); + newIDs=cell(1,nr); + end + + A=cat(2,M{i}{:}); %This day's As. + + for m=1:n2merge %merge A's by their STD deviation. + display(m) + IDs = find(MC(:, m)); + ind_del(IDs(1))= false; + + A_temp=A(:,logical(MC(:,m))); + STD_temp=STD(logical(MC(:,m))); + catSTD=STD_temp./sum(STD_temp); + + weightedA=A_temp*catSTD; + + Afinal(:,IDs(1))=weightedA; + + if i==1 + merged_ROIs{m} = IDs; + newIDs{IDs(1)} = IDs; + end + end + Afinal(:,ind_del)=[]; + Afinal_alldays{i}=Afinal; + if i==1 + newIDs(ind_del) = []; + end +end + diff --git a/BatchVerMO/merge_detail.m b/BatchVerMO/merge_detail.m new file mode 100644 index 0000000..0d08d68 --- /dev/null +++ b/BatchVerMO/merge_detail.m @@ -0,0 +1,38 @@ +function MC=merge_detail(flag_merge) +% output: MC, dim: number of old neurons * number of new neurons. +% Each column is a true or false stating if this old neuron +% should be merged into this new neuron. +% The function should be the same as the C++ import in the original CNMF-E, +% except it is all in MATLAB code now. + +%------------------- with C++ import graph_connected_comp version----------------------- +% [l,c] = graph_connected_comp(sparse(flag_merge)); % extract connected components +% MC = bsxfun(@eq, reshape(l, [],1), 1:c); +% MC(:, sum(MC,1)==1) = []; +%--------------------------------------------------------------------------------------- + +% Shijie Gu, techel@live.cn + +mergegroups={}; +for i=1:size(flag_merge,1) + ind_temp=find(flag_merge(i,:)); + if isempty(ind_temp) + continue + else + ind_temp=[i ind_temp]; + end + mergegroups_intersect = cellfun(@(x) intersect(x,ind_temp),mergegroups,'UniformOutput', false); + mergegroups_idx = find(~cellfun('isempty',mergegroups_intersect)); + if ~isempty(mergegroups_idx) + mergegroups{mergegroups_idx(1)}=unique(cat(2,ind_temp,mergegroups{mergegroups_idx})); + if length(mergegroups_idx)>1 + mergegroups(mergegroups_idx(2:end))=[]; + end + else + mergegroups{end+1}=ind_temp; + end +end +allneurons=1:size(flag_merge,1); +MC=cellfun(@(x) ismember(allneurons,x),mergegroups,'UniformOutput',false); +MC=cat(1,MC{:}); +MC=MC'; \ No newline at end of file diff --git a/BatchVerMO/merge_with_eachday.m b/BatchVerMO/merge_with_eachday.m new file mode 100644 index 0000000..95f6502 --- /dev/null +++ b/BatchVerMO/merge_with_eachday.m @@ -0,0 +1,20 @@ +function [MC_new, ind]=merge_with_eachday(M,MC) +% This function filters MC so that those MC columns that have neurons merged from +% each day will stay +% M, the M in mergeACforMo +% MC, the MC you want to filter +% ind is the index for the sequence of the original MC that gives the new +% MC_new + +eachday_nn=cellfun(@(x) size(x,2),M{1}); +MC_new=[]; +ind=[]; +for m=1:size(MC,2) + MC_current=MC(:,m); + MC_current_split=mat2cell(MC_current,eachday_nn,1); + existence_each_day=cellfun(@(x) any(x),MC_current_split); + if sum(existence_each_day)==numel(M) + MC_new(:,end+1)=MC(:,m); + end + ind=[ind,m]; +end \ No newline at end of file diff --git a/BatchVerMO/motion/BatchVerMotionSection.m b/BatchVerMO/motion/BatchVerMotionSection.m new file mode 100644 index 0000000..24adb95 --- /dev/null +++ b/BatchVerMO/motion/BatchVerMotionSection.m @@ -0,0 +1,126 @@ +%% cnmfe (BatchVer) - MotionSection +% Shijie Gu + +%% 1. Concatenate A's from all CNMFE_BatchVer.mat's +% AsfromDaysPic has pictures +% AsfromDaysCell has A's +birdnum='6922'; +cnmfedir='X:\EmilyShijieShared_old\6922_moBatchVer\'; +DatadirForA=fullfile(cnmfedir,'*PartI_Afinal*'); +Alist=dir(DatadirForA); +AnumInFolder=numel(Alist); + +AsfromDaysPic=[]; +AsfromDaysCell=cell(1,AnumInFolder); +AsfromDaysCell_central=cell(1,AnumInFolder); +sizes=[]; +for ia=1:AnumInFolder + Anext=load(fullfile(cnmfedir,Alist(ia).name)); + %ColorAllNeurons(Anext.Afinal,300,400,num2str(ia),outputdirForA) + Atemp=Anext.Afinal; + AsfromDaysCell{ia}=Atemp; % individual column is each A, for actual motion correction + + Atemp=centralA(Atemp); + AsfromDaysCell_central{ia}=Atemp; + + k = size(Atemp,2); + sizes=[sizes k]; + + AsfromDaysPic=cat(3,AsfromDaysPic,A2image(Atemp,300,400)); % whole picture, for registering, getting shifts +end + +%% 2. for each A in AsfromDays, register others to it. +Y = AsfromDaysPic; +Y = single(Y); % convert to single precision +T = size(Y,ndims(Y)); + +options_nonrigid = NoRMCorreSetParms('upd_template',false,'iter',1,... + 'd1',size(Y,1),'d2',size(Y,2),'grid_size',[80,80],'min_patch_size',[50,50,1],'overlap_pre',[10,10,1],'overlap_post',[10,10,1],... + 'mot_uf',10,'bin_width',1,... + 'shifts_method','cubic',... + 'max_shift',50,'max_dev',20,'us_fac',5,... + 'boundary','zero','iter',1); + % mot_uf: upsamling factor for + % interpolation and individual + % registration. + % us_fac: Upsampling factor (integer). Images will be registered to + % within 1/usfac of a pixel. For example usfac = 20 means the + % images will be registered within 1/20 of a pixel. (default = 1) +% Additional parameters +update_num=2; +gridstartend=[1,300,61,400,1,1]; +%% +M=cell(1,AnumInFolder); M_central=cell(1,AnumInFolder); +ind_del=cell(1,AnumInFolder); +% the only left out one; +M{1}{1}=AsfromDaysCell{1}; ind_del{1}{1}=false(1,sizes(1)); +M_central{1}{1}=centralA(AsfromDaysCell{1}); + +for ia=2:AnumInFolder + shifts_up_1=cell(1,ia-1); shifts_up_2=cell(1,ia-1); + + siz_oneday=sizes(ia); + % The day's own data + M{ia}{ia}=AsfromDaysCell{ia}; + M_central{ia}{ia}=centralA(AsfromDaysCell{ia}); + ind_del{ia}{ia}=false(1,siz_oneday); + % Previous days + M_buffer=[]; + for io=ia-1:-1:1 + if io==ia-1 + Y_template=Y(:,:,io); + else + Y_template=A2image(cat(2,M{io}{1:ia-1}),size(Y,1),size(Y,2)); + end + if io==ia-1 + Y_toregister=Y(:,:,ia); + else + Y_toregister=A2image(cat(2,M{io+1}{io+1},M{io+1}{ia}),size(Y,1),size(Y,2)); + end + + %tic; [M{ia},shifts{ia},~,xxsfyysf,ind_del{ia}] = normcorre_BatchVer(Y_ex_oneday,options_nonrigid,Y_oneday,siz_ex_oneday,As_ex_oneday,startendgrid,update_num); toc + tic; [M_temp,shifts,shifts_up_1{io},shifts_up_2{io},~,xxsfyysf,ind_del_temp] = ... + normcorre_BatchVer(Y_toregister,options_nonrigid,Y_template,siz_oneday,M{io+1}{ia},gridstartend,update_num); toc + M{io}{ia}=M_temp{1}; + M_central{io}{ia}=centralA(M_temp{1}); + ind_del{io}{ia}=ind_del_temp{1}; + + % inverse consec + shifts_up_1_tmp=sum(cat(3,shifts_up_1{io:ia-1}),3); shifts_up_2_tmp=sum(cat(3,shifts_up_2{io:ia-1}),3); + Mf_temp=[]; + ind_del{ia}{io}=false(1,sizes(io)); + for ni=1:sizes(io) + Y_one_neuron=reshape(AsfromDaysCell{io}(:,ni),size(Y,1),size(Y,2)); + Y_one_neuron(gridstartend(1):gridstartend(2),gridstartend(3):gridstartend(4),gridstartend(5):gridstartend(6)) = ... + imwarp(Y_one_neuron(gridstartend(1):gridstartend(2),gridstartend(3):gridstartend(4),gridstartend(5):gridstartend(6)),-cat(3,shifts_up_2_tmp.*(-1),shifts_up_1_tmp.*(-1)),options_nonrigid.shifts_method); + if any(Y_one_neuron)==0 + ind_del{ia}{io}(ni)=true; + end + Mf_temp=[Mf_temp reshape(Y_one_neuron,[],1)]; + end + M{ia}{io}=Mf_temp; + M_central{ia}{io}=centralA(M{ia}{io}); + end +end + +%% +ind_del_full_cell=cellfun(@(x) cell2mat(x), ind_del, 'UniformOutput',0); +ind_del_full=sum(reshape(cell2mat(ind_del_full_cell),1,sum(sizes),[]),3)>0; %sum(cat(3,ind_del{:}))>0; +ind_del_full_cell=mat2cell(~ind_del_full,1,sizes); +N_eachday=cellfun(@(x) sum(x), ind_del_full_cell); +M_concat=cellfun(@(x) cat(2,x{:}), M, 'UniformOutput',0); +M_del = cellfun(@(x) x(:,~ind_del_full), M_concat, 'UniformOutput',0); +M_final = cellfun(@(x) mat2cell(x, [size(x,1)], N_eachday), M_del, 'UniformOutput',0); + +%% +for ia=1:numel(M_final) + for ii=1:numel(M_final{ia}) + ColorAllNeuronsForMo(M_final{ia}{ii},300,400,['Day' num2str(ii) 'after Day' num2str(ia)], 'X:\EmilyShijieShared_old\6922_moBatchVer',xxsfyysf,[10,10]); + %ColorAllNeuronsForMo(A2,300,400,['18' 'Newupdating after' num2str(ia)],outputdirForA,xxsfyysf); + end +end + +M=M_final; +%% Save results +Vars = 'M_final shifts template birdnum'; +eval(sprintf('save %scnmfe_BatchVer_PartII_MotionCorrection.mat %s -v7.3', cnmfedir, Vars)); diff --git a/BatchVerMO/motion/CalText_AddText.m b/BatchVerMO/motion/CalText_AddText.m new file mode 100644 index 0000000..c96a729 --- /dev/null +++ b/BatchVerMO/motion/CalText_AddText.m @@ -0,0 +1,19 @@ +function Position=CalText_AddText(Position,A,d1,d2) + if nargin<2 + k=size(Position,2); + text(Position(1,:),Position(2,:),cellstr(num2str((1:k)'))','Color','black') + else + plotOrNot=Position; + k=size(A,2); + Atemp=reshape(A,d1,d2,k); + Position=zeros(2,k); + for i=1:k + Atemp_=Atemp(:,:,i); + [row_ind,col_ind] = find(Atemp_>0); + Position(2,i)=mean(row_ind); + Position(1,i)=mean(col_ind); + end + if plotOrNot + text(Position(1,:),Position(2,:),cellstr(num2str((1:k)'))','Color','black'); + end + end \ No newline at end of file diff --git a/BatchVerMO/motion/InterSpike.m b/BatchVerMO/motion/InterSpike.m new file mode 100644 index 0000000..088bd53 --- /dev/null +++ b/BatchVerMO/motion/InterSpike.m @@ -0,0 +1,88 @@ +acf=ce; +for i=1:size(C,1) + acf(i) = autocorr(C(i,:)) ; +end + %% + figure + subplot(3,1,1) + plot(lags{1}(3,:),r{1}(3,:)); + subplot(3,1,2) + plot(lags{2}(6,:),r{2}(6,:)); + subplot(3,1,3) + plot(lags{3}(5,:),r{3}(5,:)); + + %% + eachfilenum_cumsum=cumsum(eachdayfilenum); + filenumsum = eachfilenum_cumsum(end); + S_L=length(eachfilenum_cumsum); + for i= 1:filenumsum % file + k(i)=find((eachfilenum_cumsum>=i),1); + end + C_in_days=cell(k(end),1); + for i =1:filenumsum + C_in_days{k(i)}=[C_in_days{k(i)} ACS(i).Cin_raw]; + end + %% + r=cell(k(end),1); + lags=r; + for i =1:k(end) + r_=zeros(size(C_in_days{i},1),(size(C_in_days{i},2)*2-1)); + lags_=r_; + for j=1:size(C_in_days{i},1) + [r_(j,:),lags_(j,:)] = xcorr(C_in_days{i}(j,:)); + end + r{i}=r_; + lags{i}=lags_; + end +%% Auto-correlation +each_neuron_end=cumsum(cellfun(@(x) size(x,2),M1{1})); +each_neuron_start=[1,each_neuron_end(1:end-1)+1]; +each_neuron_number=cellfun(@(x) size(x,2),M1{1}); + +C_raw=cell(numel(each_neuron_number),1); +for n=1:filenumsum % each day's temperal trace in each day's spatial trace + d=find((eachfilenum_cumsum>=n),1); + C_raw{d}=[C_raw{d} ACS(n).Cin_raw(each_neuron_start(d):each_neuron_end(d),:)]; +end + +r=zeros(sum(each_neuron_number),neuron_full.Fs*100*2+1); +lags=r; +for d=1:numel(each_neuron_number)% do autocorrelation + for j=1:size(C_raw{d},1) + k=each_neuron_start(d); + [r(k+j-1,:),lags(k+j-1,:)] = xcorr(C_raw{d}(j,:),neuron_full.Fs*100); + end +end +%% Cross correlation +R=zeros(sum(each_neuron_number),sum(each_neuron_number)); +parfor j=1:sum(each_neuron_number) + Rj=R(j,:); + d=find((cumsum(each_neuron_number)>=j),1); + if d==1 + for k=each_neuron_start(1):each_neuron_end(d+1) + [R_,~] = xcorr(r(j,:),r(k,:),'coeff');Rj(k)=max(R_); + end + elseif d==numel(each_neuron_number) + for k=each_neuron_start(d-1):each_neuron_end(d) + [R_,~] = xcorr(r(j,:),r(k,:),'coeff');Rj(k)=max(R_); + end + else + for k=each_neuron_start(d-1):each_neuron_end(d+1) + [R_,~] = xcorr(r(j,:),r(k,:),'coeff');Rj(k)=max(R_); + end + end + R(j,:)=Rj; +end + +%% Print R +Rs_mean=zeros(1,numel(merged_ROIs)); +Rs=cell(1,numel(merged_ROIs)); +for n=1:numel(merged_ROIs) + origins=merged_ROIs{n}; + Rs_=zeros(1,numel(origins)-1); + for i=1:(numel(origins)-1) + Rs_(i)=R(origins(i),origins(i+1)); + end + Rs_mean(n)=mean(Rs_); + Rs{n}=Rs_; +end \ No newline at end of file diff --git a/BatchVerMO/motion/ManualShift.fig b/BatchVerMO/motion/ManualShift.fig new file mode 100644 index 0000000..f134582 Binary files /dev/null and b/BatchVerMO/motion/ManualShift.fig differ diff --git a/BatchVerMO/motion/ManualShift.m b/BatchVerMO/motion/ManualShift.m new file mode 100644 index 0000000..4da488a --- /dev/null +++ b/BatchVerMO/motion/ManualShift.m @@ -0,0 +1,1012 @@ +function varargout = ManualShift(varargin) +% MANUALSHIFT MATLAB code for ManualShift.fig +% MANUALSHIFT, by itself, creates a new MANUALSHIFT or raises the existing +% singleton*. +% +% H = MANUALSHIFT returns the handle to a new MANUALSHIFT or the handle to +% the existing singleton*. +% +% MANUALSHIFT('CALLBACK',hObject,eventData,handles,...) calls the local +% function named CALLBACK in MANUALSHIFT.M with the given input arguments. +% +% MANUALSHIFT('Property','Value',...) creates a new MANUALSHIFT or raises the +% existing singleton*. Starting from the left, property value pairs are +% applied to the GUI before ManualShift_OpeningFcn gets called. An +% unrecognized property name or invalid value makes property application +% stop. All inputs are passed to ManualShift_OpeningFcn via varargin. +% +% *See GUI Options on GUIDE's Tools menu. Choose "GUI allows only one +% instance to run (singleton)". +% +% See also: GUIDE, GUIDATA, GUIHANDLES + +% Edit the above text to modify the response to help ManualShift + +% Last Modified by GUIDE v2.5 21-Aug-2017 15:57:48 + + +% Begin initialization code - DO NOT EDIT +gui_Singleton = 1; +gui_State = struct('gui_Name', mfilename, ... + 'gui_Singleton', gui_Singleton, ... + 'gui_OpeningFcn', @ManualShift_OpeningFcn, ... + 'gui_OutputFcn', @ManualShift_OutputFcn, ... + 'gui_LayoutFcn', [] , ... + 'gui_Callback', []); +if nargin && ischar(varargin{1}) + gui_State.gui_Callback = str2func(varargin{1}); +end + +if nargout + [varargout{1:nargout}] = gui_mainfcn(gui_State, varargin{:}); +else + gui_mainfcn(gui_State, varargin{:}); +end +% End initialization code - DO NOT EDIT + + +% --- Executes just before ManualShift is made visible. +function ManualShift_OpeningFcn(hObject, eventdata, handles, varargin) +% This function has no output args, see OutputFcn. +% hObject handle to figure +% eventdata reserved - to be defined in a future version of MATLAB +% handles structure with handles and user data (see GUIDATA) +% varargin command line arguments to ManualShift (see VARARGIN) + +% Parsing inputs +guidata(hObject, handles); + +p = inputParser; +addParameter(p,'M',[]); +addParameter(p,'d1',300); +addParameter(p,'d2',400); +addParameter(p,'cnmfedir',[]); + +parse(p, varargin{:}); +handles.M=p.Results.M; +handles.M_intact=p.Results.M; % for cancel buttons. +handles.d1=p.Results.d1; +handles.d2=p.Results.d2; +handles.pickOrnot=false; +handles.normco.cnmfedir=p.Results.cnmfedir; % All variables related to normcorr are put in handles.normco +handles.plotgrid=false; +handles.T=[]; + +if ischar(handles.normco.cnmfedir) + cnmfedir=handles.normco.cnmfedir; %loadAs will use 'cnmfedir' + loadAs + if or(isempty(AsfromDaysCell),isempty(AsfromDaysPic)) + error('No results for input.') + end + handles.normco.AsfromDaysPic=AsfromDaysPic; + handles.normco.AsfromDaysCell=AsfromDaysCell; + + handles=Update_Plot_Raw(1,handles); + + % Opening look if M does not exist + set(findall(handles.manualpanel, '-property', 'enable'), 'enable', 'off') + set(handles.slider_auto,'Min',1); + set(handles.slider_auto,'Max',numel(AsfromDaysCell)); + set(handles.slider_auto,'Value',1); + set(handles.slider_auto,'SliderStep', [1/(numel(AsfromDaysCell)-1), 1/(numel(AsfromDaysCell)-1)]); +else + % All pairs of alignment + M=p.Results.M; + + % Opening look if M exists + handles=Precalculate_Minfo(M,handles); + + % Opening look + % axes-creat the overlay picture + handles=Update_Plot(1,handles); + linkaxes([handles.axes1 handles.axes2]) + +end + + +% Choose default command line output for ManualShift +handles.output = hObject; + +% Update handles structure +guidata(hObject, handles); + +% UIWAIT makes ManualShift wait for user response (see UIRESUME) +uiwait(handles.figure1); + + +% --- Outputs from this function are returned to the command line. +function varargout = ManualShift_OutputFcn(hObject, eventdata, handles) +% varargout cell array for returning output args (see VARARGOUT); +% hObject handle to figure +% eventdata reserved - to be defined in a future version of MATLAB +% handles structure with handles and user data (see GUIDATA) + +% Get default command line output from handles structure +display('Saving results...') +varargout{1} = handles.M; +varargout{2} = handles.T; +varargout{3} = handles.output; +varargout{4} = handles.M_intact; +delete(handles.figure1); + +function neuron_list_tmpl_Callback(hObject, eventdata, handles) +% hObject handle to neuron_list_tmpl (see GCBO) +% eventdata reserved - to be defined in a future version of MATLAB +% handles structure with handles and user data (see GUIDATA) + +% Hints: get(hObject,'String') returns contents of neuron_list_tmpl as text +% str2double(get(hObject,'String')) returns contents of neuron_list_tmpl as a double +neuron_list_str=get(hObject,'String'); +display(neuron_list_str) +if ~strcmp(neuron_list_str(end),';') + neuron_list_str(end+1)=';'; +end +neuron_list=sscanf(neuron_list_str,'%d-%d;'); +handles.neuron_list=reshape(neuron_list,2,[]); %two rows, first row-template, second row-the toAlign. +% Update handles structure +guidata(hObject, handles); + + +% --- Executes during object creation, after setting all properties. +function neuron_list_tmpl_CreateFcn(hObject, eventdata, handles) +% hObject handle to neuron_list_tmpl (see GCBO) +% eventdata reserved - to be defined in a future version of MATLAB +% handles empty - handles not created until after all CreateFcns called + +% Hint: edit controls usually have a white background on Windows. +% See ISPC and COMPUTER. +if ispc && isequal(get(hObject,'BackgroundColor'), get(0,'defaultUicontrolBackgroundColor')) + set(hObject,'BackgroundColor','white'); +end + + +% --- Executes on slider movement. +function slider1_Callback(hObject, eventdata, handles) +% hObject handle to slider1 (see GCBO) +% eventdata reserved - to be defined in a future version of MATLAB +% handles structure with handles and user data (see GUIDATA) + +% Hints: get(hObject,'Value') returns position of slider +% get(hObject,'Min') and get(hObject,'Max') to determine range of slider + +currentpair_ind=get(hObject,'Value'); +currentpair_ind = round(currentpair_ind); %round off this value +set(hObject, 'Value', currentpair_ind); + +handles=Update_Plot(currentpair_ind,handles); +guidata(hObject, handles); + +% --- Executes during object creation, after setting all properties. +function slider1_CreateFcn(hObject, eventdata, handles) +% hObject handle to slider1 (see GCBO) +% eventdata reserved - to be defined in a future version of MATLAB +% handles empty - handles not created until after all CreateFcns called + +% Hint: slider controls usually have a light gray background. +if isequal(get(hObject,'BackgroundColor'), get(0,'defaultUicontrolBackgroundColor')) + set(hObject,'BackgroundColor',[.9 .9 .9]); +end +set(hObject,'Value',1); + + +function ToAlign_Callback(hObject, eventdata, handles) +% hObject handle to ToAlign (see GCBO) +% eventdata reserved - to be defined in a future version of MATLAB +% handles structure with handles and user data (see GUIDATA) + +% Hints: get(hObject,'String') returns contents of ToAlign as text +% str2double(get(hObject,'String')) returns contents of ToAlign as a double +ToAlign_text=get(hObject,'String'); +ToAlign_num=sscanf(ToAlign_text,'%d-%d'); +handles.currentpair(1)=ToAlign_num(1); +handles.currentpair(2)=ToAlign_num(2); +handles=Update_Plot([],handles); +guidata(hObject, handles); + +% --- Executes during object creation, after setting all properties. +function ToAlign_CreateFcn(hObject, eventdata, handles) +% hObject handle to ToAlign (see GCBO) +% eventdata reserved - to be defined in a future version of MATLAB +% handles empty - handles not created until after all CreateFcns called + +% Hint: edit controls usually have a white background on Windows. +% See ISPC and COMPUTER. +if ispc && isequal(get(hObject,'BackgroundColor'), get(0,'defaultUicontrolBackgroundColor')) + set(hObject,'BackgroundColor','white'); +end +set(hObject,'String','1-2') + + +% --- Executes on button press in pushbutton1. +function pushbutton1_Callback(hObject, eventdata, handles) +% hObject handle to pushbutton1 (see GCBO) +% eventdata reserved - to be defined in a future version of MATLAB +% handles structure with handles and user data (see GUIDATA) + +neuron_list_tmpl_Callback(handles.neuron_list_tmpl,eventdata, handles) +handles=guidata(hObject); +handles.M_intact=handles.M; +d1=handles.d1; d2=handles.d2; +template_num=handles.currentpair(1); ToAlign_num=handles.currentpair(2); +neuron_templ_ind=handles.neuron_list(1,:); neuron_ToAlign_ind=handles.neuron_list(2,:); + +for i = 1:length(neuron_templ_ind) + template_neuron_A=handles.M{template_num}{template_num}(:,neuron_templ_ind(i)); + ToAlign_neuron_A=handles.M{template_num}{ToAlign_num}(:,neuron_ToAlign_ind(i)); + % template_neuron_center = round(com(template_neuron_A, d1, d2)); + % ToAlign_neuron_center = round(com(ToAlign_neuron_A, d1, d2)); + % displacement_vector=[ToAlign_neuron_center template_neuron_center]; + [D,~] = imregdemons(A2image(ToAlign_neuron_A,d1,d2),A2image(template_neuron_A,d1,d2)); + handles.M{template_num}{ToAlign_num}(:,neuron_ToAlign_ind(i))=... + reshape(imwarp(reshape(ToAlign_neuron_A,d1,d2),D,'cubic'),[],1); + + template_neuron_A_reverse=handles.M{ToAlign_num}{template_num}(:,neuron_templ_ind(i)); + handles.M{ToAlign_num}{template_num}(:,neuron_templ_ind(i))=... + reshape(imwarp(reshape(template_neuron_A_reverse,d1,d2),D.*(-1),'cubic'),[],1); +end +display('A Updated') +guidata(hObject,handles); +ToAlign_Callback(handles.ToAlign, eventdata, handles) +% Update handles structure +%guidata(hObject, handles); + + +% --- Executes on button press in pushbutton4. +function pushbutton4_Callback(hObject, eventdata, handles) +% hObject handle to pushbutton4 (see GCBO) +% eventdata reserved - to be defined in a future version of MATLAB +% handles structure with handles and user data (see GUIDATA) +template_num=handles.currentpair(1); ToAlign_num=handles.currentpair(2); +neuron_templ_ind=handles.neuron_list(1,:); neuron_ToAlign_ind=handles.neuron_list(2,:); + +for i = 1:length(neuron_templ_ind) + + handles.M{template_num}{ToAlign_num}(:,neuron_ToAlign_ind(i))=... + handles.M_intact{template_num}{ToAlign_num}(:,neuron_ToAlign_ind(i)); + handles.M{ToAlign_num}{template_num}(:,neuron_templ_ind(i))=... + handles.M_intact{ToAlign_num}{template_num}(:,neuron_templ_ind(i)); +end + +% Update handles structure +guidata(hObject, handles); +ToAlign_Callback(handles.ToAlign, eventdata, handles) + +% --- Executes during object creation, after setting all properties. +function axes1_CreateFcn(hObject, eventdata, handles) +% hObject handle to axes1 (see GCBO) +% eventdata reserved - to be defined in a future version of MATLAB +% handles empty - handles not created until after all CreateFcns called + +% Hint: place code in OpeningFcn to populate axes1 +set(gca,'tag','axes1') + +% --- Executes during object creation, after setting all properties. +function axes2_CreateFcn(hObject, eventdata, handles) +% hObject handle to axes2 (see GCBO) +% eventdata reserved - to be defined in a future version of MATLAB +% handles empty - handles not created until after all CreateFcns called + +% Hint: place code in OpeningFcn to populate axes2 +set(gca,'tag','axes2') + +% --- Executes on key press with focus on figure1 and none of its controls. +function figure1_KeyPressFcn(hObject, eventdata, handles) +% hObject handle to figure1 (see GCBO) +% eventdata structure with the following fields (see MATLAB.UI.FIGURE) +% Key: name of the key that was pressed, in lower case +% Character: character interpretation of the key(s) that was pressed +% Modifier: name(s) of the modifier key(s) (i.e., control, shift) pressed +% handles structure with handles and user data (see GUIDATA) +% KeyPressed = eventdata.Key; +% switch KeyPressed +% case 'rightarrow' +% axes(handles.axes1) +% imshow(handles.B) +% case 'leftarrow' +% axes(handles.axes1) +% imshow(handles.A) +% case 'downarrow' +% axes(handles.axes1) +% imshowpair(handles.A,handles.B,'falsecolor','Scaling','independent'); +% case 'uparrow' +% set(hObject,'CurrentObject',handles.slider1) +% end + + +% --- Executes on key press with focus on slider1 and none of its controls. +function slider1_KeyPressFcn(hObject, eventdata, handles) +% hObject handle to slider1 (see GCBO) +% eventdata structure with the following fields (see MATLAB.UI.CONTROL.UICONTROL) +% Key: name of the key that was pressed, in lower case +% Character: character interpretation of the key(s) that was pressed +% Modifier: name(s) of the modifier key(s) (i.e., control, shift) pressed +% handles structure with handles and user data (see GUIDATA) + + + +% --- Executes on key press with focus on figure1 or any of its controls. +function figure1_WindowKeyPressFcn(hObject, eventdata, handles) +% hObject handle to figure1 (see GCBO) +% eventdata structure with the following fields (see MATLAB.UI.FIGURE) +% Key: name of the key that was pressed, in lower case +% Character: character interpretation of the key(s) that was pressed +% Modifier: name(s) of the modifier key(s) (i.e., control, shift) pressed +% handles structure with handles and user data (see GUIDATA) +if ~isempty(handles.M) +uiresume(hObject); +d1=handles.d1; +d2=handles.d2; +KeyPressed = eventdata.Key; +switch KeyPressed + case 'space' + slider_enable=get(handles.slider1,'Enable'); + if strcmp(slider_enable,'on') + set(handles.slider1, 'Enable', 'off'); + uicontrol(handles.figure1) + elseif strcmp(slider_enable,'off') + set(handles.slider1, 'Enable', 'on'); + uicontrol(handles.slider1) + end + case 'rightarrow' + curr_focus=gco(handles.figure1); + try + tag=curr_focus.Tag; + catch + tag=[]; + end + if ~strcmp(tag,'slider1') + handles.mode='toAlign'; + axes(gca) + current_axes=gca; + if strcmp(current_axes.Tag,'axes1') + imshow(handles.B) + CalText_AddText(handles.B_text); + handles.mode='toAlign'; + set(gca,'tag','axes1') + set(handles.text6,'String','Registered New A To Template'); + if handles.plotgrid==true + try [handles.normco.h1,handles.normco.h2]=plot_grid(handles.normco.xxsfyysf,handles.normco.overlap,d1,d2); + catch; disp('No grid information loaded.'); end + end + elseif strcmp(current_axes.Tag,'axes2') + imshow(handles.D) + CalText_AddText(handles.D_text); + handles.mode='toAlign'; + set(gca,'tag','axes2'); + set(handles.text7,'String','Original A to Align'); + if handles.plotgrid==true + try [handles.normco.h3,handles.normco.h4]=plot_grid(handles.normco.xxsfyysf,handles.normco.overlap,d1,d2); + catch; disp('No grid information loaded.'); end + end + end + + end + case 'leftarrow' + curr_focus=gco(handles.figure1); + try + tag=curr_focus.Tag; + catch + tag=[]; + end + if ~strcmp(tag,'slider1') + handles.mode='template'; + axes(gca) + current_axes=gca; + if strcmp(current_axes.Tag,'axes1') + imshow(handles.A); CalText_AddText(handles.A_text); + set(gca,'tag','axes1'); set(handles.text6,'String','Template'); + if handles.plotgrid==true + try [handles.normco.h1,handles.normco.h2]=plot_grid(handles.normco.xxsfyysf,handles.normco.overlap,d1,d2); + catch; disp('No grid information loaded.'); end + end + elseif strcmp(current_axes.Tag,'axes2') + imshow(handles.A); CalText_AddText(handles.A_text); + set(gca,'tag','axes2'); set(handles.text7,'String','Template'); + handles.figure1.CurrentAxes=handles.axes2; + if handles.plotgrid==true + try [handles.normco.h3,handles.normco.h4]=plot_grid(handles.normco.xxsfyysf,handles.normco.overlap,d1,d2); + catch; disp('No grid information loaded.'); end + end + end + end + + case 'downarrow' + curr_focus=gco(handles.figure1); + try + tag=curr_focus.Tag; + catch + tag=[]; + end + if ~strcmp(tag,'slider1') + handles.mode='overlapping'; + axes(gca) + current_axes=gca; + if strcmp(current_axes.Tag,'axes1') + imshow(handles.C); + CalText_AddText(handles.A_text); CalText_AddText(handles.B_text); + %imshowpair(handles.A_b,handles.B_b,'falsecolor','Scaling','independent'); + set(gca,'tag','axes1') + set(handles.text6,'String','Registered(Green) and Template(Red)'); + if handles.plotgrid==true + try [handles.normco.h1,handles.normco.h2]=plot_grid(handles.normco.xxsfyysf,handles.normco.overlap,d1,d2); + catch; disp('No grid information loaded.'); end + end + elseif strcmp(current_axes.Tag,'axes2') + imshow(handles.E); + CalText_AddText(handles.A_text); CalText_AddText(handles.D_text); + %imshowpair(handles.A_b,handles.D_b,'falsecolor','Scaling','independent'); + set(gca,'tag','axes2'); + set(handles.text7,'String','Un-Registered(Green) and Template(Red)'); + handles.figure1.CurrentAxes=handles.axes2; + if handles.plotgrid==true + try [handles.normco.h3,handles.normco.h4]=plot_grid(handles.normco.xxsfyysf,handles.normco.overlap,d1,d2); + catch; disp('No grid information loaded.'); end + end + end + end + + case 'a' + set(findall(handles.autopanel, '-property', 'enable'), 'enable', 'on') + set(findall(handles.manualpanel, '-property', 'enable'), 'enable', 'off') + case 'm' + set(findall(handles.autopanel, '-property', 'enable'), 'enable', 'off') + set(findall(handles.manualpanel, '-property', 'enable'), 'enable', 'on') + set(handles.text6,'String','Registered and Template') + set(handles.slider1, 'Enable', 'off'); + case 'b' + set(findall(handles.autopanel, '-property', 'enable'), 'enable', 'on') + set(findall(handles.manualpanel, '-property', 'enable'), 'enable', 'on') + set(handles.slider1, 'Enable', 'off'); +end +guidata(hObject, handles); +uiwait(hObject); +end + +function handles=Update_Plot(currentpair_ind,handles) +d1=handles.d1; d2=handles.d2; + +if ~isempty(handles.M) + handles.mode='overlapping'; + if ~isempty(currentpair_ind) + handles.currentpair=handles.allpairs(currentpair_ind,:); + end + template_num=handles.currentpair(1); + ToAlign_num=handles.currentpair(2); + set(handles.ToAlign,'String',[num2str(template_num) '-' num2str(ToAlign_num)]); + set(handles.text10,'String',[]); + [~,currentpair_ind,~] = intersect(handles.allpairs,handles.currentpair,'rows'); + if ~isempty(currentpair_ind) + set(handles.slider1,'Value',currentpair_ind) + end + set(handles.neuron_list_tmpl,'String',[]); + + M=handles.M; + A=A2image(M{template_num}{template_num},d1,d2,false,'magenta'); handles.A=A; + A_b=A2image(M{template_num}{template_num},d1,d2,false); % black and white for imshowpair + handles.A_b=A_b; + A_text=CalText_AddText(false,M{template_num}{template_num},d1,d2); handles.A_text=A_text; + + B=A2image(M{template_num}{ToAlign_num},d1,d2,false,'green'); handles.B=B; + B_b=A2image(M{template_num}{ToAlign_num},d1,d2,false); B_b = imhistmatch(B_b,A_b); handles.B_b=B_b; + B_text=CalText_AddText(false,M{template_num}{ToAlign_num},d1,d2); handles.B_text=B_text; + + D=A2image(M{ToAlign_num}{ToAlign_num},d1,d2,false,'green'); handles.D=D; + D_b=A2image(M{ToAlign_num}{ToAlign_num},d1,d2,false); D_b = imhistmatch(D_b,A_b); handles.D_b=D_b; + D_text=CalText_AddText(false,M{ToAlign_num}{ToAlign_num},d1,d2); handles.D_text=D_text; + + axes(handles.axes1) + % imshowpair(A_b,B_b,'falsecolor','Scaling','independent') + C=imfuse(A_b,B_b,'falsecolor','Scaling','independent'); handles.C=C; + imshow(C); + CalText_AddText(A_text); + CalText_AddText(B_text); + if handles.plotgrid==true + try [handles.normco.h1,handles.normco.h2]=plot_grid(handles.normco.xxsfyysf,handles.normco.overlap,d1,d2); + catch; disp('No grid information loaded.'); end + end + set(gca,'tag','axes1') + axis tight + + axes(handles.axes2) + %imshowpair(A_b,D_b,'falsecolor','Scaling','independent') + E=imfuse(A_b,D_b,'falsecolor','Scaling','independent'); handles.E=E; + imshow(E); + CalText_AddText(A_text); + CalText_AddText(D_text); + if handles.plotgrid==true + try [handles.normco.h3,handles.normco.h4]=plot_grid(handles.normco.xxsfyysf,handles.normco.overlap,d1,d2); + catch; disp('No grid information loaded.'); end + end + set(gca,'tag','axes2') + axis tight +else + if ~isempty(currentpair_ind) + handles.normco.currentA_ind=currentpair_ind; + set(handles.slider1,'Value',currentpair_ind) + end + + handles.normco.currentA_Pic=handles.normco.AsfromDaysPic(:,:,handles.normco.currentA_ind); + axes(handles.axes1) + imshow(handles.normco.currentA_Pic); + if handles.plotgrid==true + try [handles.normco.h1,handles.normco.h2]=plot_grid(handles.normco.xxsfyysf,handles.normco.overlap,d1,d2); + catch; disp('No grid information loaded.'); end + end +end + +function handles=Update_Plot_Raw(currentpair_ind,handles) +d1=handles.d1; d2=handles.d2; + if ~isempty(currentpair_ind) + handles.normco.currentA_ind=currentpair_ind; + set(handles.slider1,'Value',currentpair_ind) + end + + handles.normco.currentA_Pic=handles.normco.AsfromDaysPic(:,:,handles.normco.currentA_ind); + axes(handles.axes1) + imshow(handles.normco.currentA_Pic); + if handles.plotgrid==true + [handles.normco.h1,handles.normco.h2]=plot_grid(handles.normco.xxsfyysf,handles.normco.overlap,d1,d2); + %catch; disp('No grid information loaded.'); end + end + +% --- Executes on mouse press over figure background. +function figure1_ButtonDownFcn(hObject, eventdata, handles) +% hObject handle to figure1 (see GCBO) +% eventdata reserved - to be defined in a future version of MATLAB +% handles structure with handles and user data (see GUIDATA) +% uiresume(hObject); +% uicontrol(hObject) +% uiwait(hObject); + +% --- Executes on mouse press over figure background, over a disabled or +% --- inactive control, or over an axes background. +function figure1_WindowButtonDownFcn(hObject, eventdata, handles) +% hObject handle to figure1 (see GCBO) +% eventdata reserved - to be defined in a future version of MATLAB +% handles structure with handles and user data (see GUIDATA) +if ~isempty(handles.M) + uiresume(hObject); + + cursorPoint = get(gca, 'CurrentPoint'); + current_axes=gca; + x=round(cursorPoint(1,1,1)); y=round(cursorPoint(1,2,1)); + display(['You picked pixel x=' num2str(x) ' y=' num2str(y)]) + d1=handles.d1; d2=handles.d2; M=handles.M; + if and(x<=d2,y<=d1) + mouse_location = sub2ind([d1 d2], y, x); + template_num=handles.currentpair(1);ToAlign_num=handles.currentpair(2); + if strcmp(handles.mode,'template') + A_=M{template_num}{template_num}; + elseif strcmp(handles.mode,'toAlign') + if strcmp(current_axes.Tag,'axes1') + A_=M{template_num}{ToAlign_num}; + elseif strcmp(current_axes.Tag,'axes2') + A_=M{ToAlign_num}{ToAlign_num}; + end + else + disp('You are in "overlapping" mode.') + end + try + neuron_ind=find(A_(mouse_location,:)>0); + set(handles.text10,'String',['neuron #' num2str(neuron_ind)]) + catch + disp('Neuron picking is not supported in overlapping mode.') + end + %guidata(hObject, handles); + end +%end +uiwait(hObject); +end + + +% --- Executes when user attempts to close figure1. +function figure1_CloseRequestFcn(hObject, eventdata, handles) +% hObject handle to figure1 (see GCBO) +% eventdata reserved - to be defined in a future version of MATLAB +% handles structure with handles and user data (see GUIDATA) + +% Hint: delete(hObject) closes the figure +if isequal(get(hObject, 'waitstatus'), 'waiting') +% The GUI is still in UIWAIT, us UIRESUME +uiresume(hObject); +else +% The GUI is no longer waiting, just close it +delete(hObject); +end + +function neuron_list_tmpl_KeyPressFcn(hObject, eventdata, handles) + + + +function edit_grid_size_Callback(hObject, eventdata, handles) +% hObject handle to edit_grid_size (see GCBO) +% eventdata reserved - to be defined in a future version of MATLAB +% handles structure with handles and user data (see GUIDATA) + +% Hints: get(hObject,'String') returns contents of edit_grid_size as text +% str2double(get(hObject,'String')) returns contents of edit_grid_size as a double + + +% --- Executes during object creation, after setting all properties. +function edit_grid_size_CreateFcn(hObject, eventdata, handles) +% hObject handle to edit_grid_size (see GCBO) +% eventdata reserved - to be defined in a future version of MATLAB +% handles empty - handles not created until after all CreateFcns called + +% Hint: edit controls usually have a white background on Windows. +% See ISPC and COMPUTER. +if ispc && isequal(get(hObject,'BackgroundColor'), get(0,'defaultUicontrolBackgroundColor')) + set(hObject,'BackgroundColor','white'); +end + + + +function edit_min_patch_size_Callback(hObject, eventdata, handles) +% hObject handle to edit_min_patch_size (see GCBO) +% eventdata reserved - to be defined in a future version of MATLAB +% handles structure with handles and user data (see GUIDATA) + +% Hints: get(hObject,'String') returns contents of edit_min_patch_size as text +% str2double(get(hObject,'String')) returns contents of edit_min_patch_size as a double + + +% --- Executes during object creation, after setting all properties. +function edit_min_patch_size_CreateFcn(hObject, eventdata, handles) +% hObject handle to edit_min_patch_size (see GCBO) +% eventdata reserved - to be defined in a future version of MATLAB +% handles empty - handles not created until after all CreateFcns called + +% Hint: edit controls usually have a white background on Windows. +% See ISPC and COMPUTER. +if ispc && isequal(get(hObject,'BackgroundColor'), get(0,'defaultUicontrolBackgroundColor')) + set(hObject,'BackgroundColor','white'); +end + + + +function edit_overlap_Callback(hObject, eventdata, handles) +% hObject handle to edit_overlap (see GCBO) +% eventdata reserved - to be defined in a future version of MATLAB +% handles structure with handles and user data (see GUIDATA) + +% Hints: get(hObject,'String') returns contents of edit_overlap as text +% str2double(get(hObject,'String')) returns contents of edit_overlap as a double + + +% --- Executes during object creation, after setting all properties. +function edit_overlap_CreateFcn(hObject, eventdata, handles) +% hObject handle to edit_overlap (see GCBO) +% eventdata reserved - to be defined in a future version of MATLAB +% handles empty - handles not created until after all CreateFcns called + +% Hint: edit controls usually have a white background on Windows. +% See ISPC and COMPUTER. +if ispc && isequal(get(hObject,'BackgroundColor'), get(0,'defaultUicontrolBackgroundColor')) + set(hObject,'BackgroundColor','white'); +end + + + +function edit_mot_uf_Callback(hObject, eventdata, handles) +% hObject handle to edit_mot_uf (see GCBO) +% eventdata reserved - to be defined in a future version of MATLAB +% handles structure with handles and user data (see GUIDATA) + +% Hints: get(hObject,'String') returns contents of edit_mot_uf as text +% str2double(get(hObject,'String')) returns contents of edit_mot_uf as a double + + +% --- Executes during object creation, after setting all properties. +function edit_mot_uf_CreateFcn(hObject, eventdata, handles) +% hObject handle to edit_mot_uf (see GCBO) +% eventdata reserved - to be defined in a future version of MATLAB +% handles empty - handles not created until after all CreateFcns called + +% Hint: edit controls usually have a white background on Windows. +% See ISPC and COMPUTER. +if ispc && isequal(get(hObject,'BackgroundColor'), get(0,'defaultUicontrolBackgroundColor')) + set(hObject,'BackgroundColor','white'); +end + + + +function edit_max_shift_Callback(hObject, eventdata, handles) +% hObject handle to edit_max_shift (see GCBO) +% eventdata reserved - to be defined in a future version of MATLAB +% handles structure with handles and user data (see GUIDATA) + +% Hints: get(hObject,'String') returns contents of edit_max_shift as text +% str2double(get(hObject,'String')) returns contents of edit_max_shift as a double + + +% --- Executes during object creation, after setting all properties. +function edit_max_shift_CreateFcn(hObject, eventdata, handles) +% hObject handle to edit_max_shift (see GCBO) +% eventdata reserved - to be defined in a future version of MATLAB +% handles empty - handles not created until after all CreateFcns called + +% Hint: edit controls usually have a white background on Windows. +% See ISPC and COMPUTER. +if ispc && isequal(get(hObject,'BackgroundColor'), get(0,'defaultUicontrolBackgroundColor')) + set(hObject,'BackgroundColor','white'); +end + + + +function edit_max_dev_Callback(hObject, eventdata, handles) +% hObject handle to edit_max_dev (see GCBO) +% eventdata reserved - to be defined in a future version of MATLAB +% handles structure with handles and user data (see GUIDATA) + +% Hints: get(hObject,'String') returns contents of edit_max_dev as text +% str2double(get(hObject,'String')) returns contents of edit_max_dev as a double + + +% --- Executes during object creation, after setting all properties. +function edit_max_dev_CreateFcn(hObject, eventdata, handles) +% hObject handle to edit_max_dev (see GCBO) +% eventdata reserved - to be defined in a future version of MATLAB +% handles empty - handles not created until after all CreateFcns called + +% Hint: edit controls usually have a white background on Windows. +% See ISPC and COMPUTER. +if ispc && isequal(get(hObject,'BackgroundColor'), get(0,'defaultUicontrolBackgroundColor')) + set(hObject,'BackgroundColor','white'); +end + + + +function edit_us_fac_Callback(hObject, eventdata, handles) +% hObject handle to edit_us_fac (see GCBO) +% eventdata reserved - to be defined in a future version of MATLAB +% handles structure with handles and user data (see GUIDATA) + +% Hints: get(hObject,'String') returns contents of edit_us_fac as text +% str2double(get(hObject,'String')) returns contents of edit_us_fac as a double + + +% --- Executes during object creation, after setting all properties. +function edit_us_fac_CreateFcn(hObject, eventdata, handles) +% hObject handle to edit_us_fac (see GCBO) +% eventdata reserved - to be defined in a future version of MATLAB +% handles empty - handles not created until after all CreateFcns called + +% Hint: edit controls usually have a white background on Windows. +% See ISPC and COMPUTER. +if ispc && isequal(get(hObject,'BackgroundColor'), get(0,'defaultUicontrolBackgroundColor')) + set(hObject,'BackgroundColor','white'); +end + + +% --- Executes on button press in SetParms. +function SetParms_Callback(hObject, eventdata, handles) +% hObject handle to SetParms (see GCBO) +% eventdata reserved - to be defined in a future version of MATLAB +% handles structure with handles and user data (see GUIDATA) +d1=handles.d1; d2=handles.d2; +Names={'mot_uf','max_shift','max_dev','us_fac'}; +for j = 1:length(Names) + eval(sprintf('%s_str=get(handles.edit_%s,''String'')',Names{j},Names{j})); +end +overlap_post=handles.normco.overlap; +overlap_pre=handles.normco.overlap; + +mot_uf=str2double(mot_uf_str); +max_shift=str2double(max_shift_str); +max_dev=str2double(max_dev_str); +us_fac=str2double(us_fac_str); + +Names={'grid_size','min_patch_size','overlap','gridstartend'}; +for j = 1:length(Names) + eval(sprintf('%s_str=get(handles.edit_%s,''String'');',Names{j},Names{j})); +end +grid_size=reshape(sscanf(grid_size_str,'%d,%d'),1,[]); +min_patch_size=[reshape(sscanf(min_patch_size_str,'%d,%d'),1,[]),1]; + +handles.normco.options_nonrigid = NoRMCorreSetParms('upd_template',false,'iter',1,... + 'd1',d1,'d2',d2,'grid_size',grid_size,'min_patch_size',min_patch_size,'overlap_pre',overlap_pre,'overlap_post',overlap_post,... + 'mot_uf',mot_uf,'bin_width',1,... + 'shifts_method','cubic',... + 'max_shift',max_shift,'max_dev',max_dev,'us_fac',us_fac,... + 'boundary','zero','iter',1); +% Update handles structure +guidata(hObject, handles); + +% --- Executes on button press in automove. +function automove_Callback(hObject, eventdata, handles) +% hObject handle to automove (see GCBO) +% eventdata reserved - to be defined in a future version of MATLAB +% handles structure with handles and user data (see GUIDATA) +[handles.M,~,handle.T]=motioncorrection(handles.normco.AsfromDaysCell,... + handles.normco.AsfromDaysPic,handles.normco.options_nonrigid,handles.normco.gridstartend); +handles.M_intact=handles.M; +handles=Precalculate_Minfo(handles.M,handles); + +% Opening look +% axes-creat the overlay picture +handles=Update_Plot(1,handles); +display('Auto Motion Correction Done.') +linkaxes([handles.axes1 handles.axes2]) +uicontrol(handles.figure1) +% Update handles structure +guidata(hObject, handles); + +% --- Executes on button press in automove_cancel. +function automove_cancel_Callback(hObject, eventdata, handles) +% hObject handle to automove_cancel (see GCBO) +% eventdata reserved - to be defined in a future version of MATLAB +% handles structure with handles and user data (see GUIDATA) +handles.M=handles.M_intact; +handles.normco.xxsfyysf=handles.normco.xxsfyysf_intact; +% Update handles structure +guidata(hObject, handles); + + + +function edit_gridstartend_Callback(hObject, eventdata, handles) +% hObject handle to edit_gridstartend (see GCBO) +% eventdata reserved - to be defined in a future version of MATLAB +% handles structure with handles and user data (see GUIDATA) + +% Hints: get(hObject,'String') returns contents of edit_gridstartend as text +% str2double(get(hObject,'String')) returns contents of edit_gridstartend as a double + + +% --- Executes during object creation, after setting all properties. +function edit_gridstartend_CreateFcn(hObject, eventdata, handles) +% hObject handle to edit_gridstartend (see GCBO) +% eventdata reserved - to be defined in a future version of MATLAB +% handles empty - handles not created until after all CreateFcns called + +% Hint: edit controls usually have a white background on Windows. +% See ISPC and COMPUTER. +if ispc && isequal(get(hObject,'BackgroundColor'), get(0,'defaultUicontrolBackgroundColor')) + set(hObject,'BackgroundColor','white'); +end + + +% --- Executes on button press in plotgrid. +function plotgrid_Callback(hObject, eventdata, handles) +% hObject handle to plotgrid (see GCBO) +% eventdata reserved - to be defined in a future version of MATLAB +% handles structure with handles and user data (see GUIDATA) +handles.plotgrid=true; +d1=handles.d1; d2=handles.d2; + + +Names={'grid_size','min_patch_size','overlap','gridstartend'}; +for j = 1:length(Names) + eval(sprintf('%s_str=get(handles.edit_%s,''String'');',Names{j},Names{j})); +end + +grid_size=reshape(sscanf(grid_size_str,'%d,%d'),1,[]); +display(grid_size) + +min_patch_size=[reshape(sscanf(min_patch_size_str,'%d,%d'),1,[]),1]; + +overlap_pre=[reshape(sscanf(overlap_str,'%d,%d'),1,[]),1]; +handles.normco.overlap=overlap_pre; + +gridstartend=[reshape(sscanf(gridstartend_str,'%d,%d,%d,%d'),1,[]),1,1]; +handles.normco.gridstartend=gridstartend; + +[xx_s,xx_f,yy_s,yy_f,~,~,~,~,~,~,~,~] = construct_grid([grid_size,1],[1 1 1],d1,d2,1,min_patch_size,gridstartend); +xxsfyysf{1}=xx_s; xxsfyysf{2}=xx_f; xxsfyysf{3}=yy_s; xxsfyysf{4}=yy_f; +handles.normco.xxsfyysf=xxsfyysf; +handles.normco.xxsfyysf_intact=xxsfyysf; + +if isfield(handles.normco,'h1') + delete(handles.normco.h1); delete(handles.normco.h2); + delete(handles.normco.h3); delete(handles.normco.h4); +end + +axes(handles.axes1) +hold on +[handles.normco.h1,handles.normco.h2]=plot_grid(handles.normco.xxsfyysf,handles.normco.overlap,d1,d2); +axes(handles.axes2) +hold on +[handles.normco.h3,handles.normco.h4]=plot_grid(handles.normco.xxsfyysf,handles.normco.overlap,d1,d2); +% Update handles structure +guidata(hObject, handles); + +% --- Executes on button press in notplotgrid. +function notplotgrid_Callback(hObject, eventdata, handles) +% hObject handle to notplotgrid (see GCBO) +% eventdata reserved - to be defined in a future version of MATLAB +% handles structure with handles and user data (see GUIDATA) +handles.plotgrid=false; + +% Update handles structure +guidata(hObject, handles); + + +% --- Executes on slider movement. +function slider_auto_Callback(hObject, eventdata, handles) +% hObject handle to slider_auto (see GCBO) +% eventdata reserved - to be defined in a future version of MATLAB +% handles structure with handles and user data (see GUIDATA) + +% Hints: get(hObject,'Value') returns position of slider +% get(hObject,'Min') and get(hObject,'Max') to determine range of slider +currentpair_ind=get(hObject,'Value'); +currentpair_ind = round(currentpair_ind); %round off this value +set(hObject, 'Value', currentpair_ind); +set(handles.text6, 'String', ['Day ' num2str(currentpair_ind) '''s Raw A']) +handles=Update_Plot_Raw(currentpair_ind,handles); +guidata(hObject, handles); + +% --- Executes during object creation, after setting all properties. +function slider_auto_CreateFcn(hObject, eventdata, handles) +% hObject handle to slider_auto (see GCBO) +% eventdata reserved - to be defined in a future version of MATLAB +% handles empty - handles not created until after all CreateFcns called + +% Hint: slider controls usually have a light gray background. +if isequal(get(hObject,'BackgroundColor'), get(0,'defaultUicontrolBackgroundColor')) + set(hObject,'BackgroundColor',[.9 .9 .9]); +end +set(hObject, 'Value', 1); + + + +function edit_del_Callback(hObject, eventdata, handles) +% hObject handle to edit_del (see GCBO) +% eventdata reserved - to be defined in a future version of MATLAB +% handles structure with handles and user data (see GUIDATA) + +% Hints: get(hObject,'String') returns contents of edit_del as text +% str2double(get(hObject,'String')) returns contents of edit_del as a double +temp_neuron_list_str=get(hObject,'String'); +display(temp_neuron_list_str) +if ~strcmp(temp_neuron_list_str(end),';') + temp_neuron_list_str(end+1)=';'; +end +temp_neuron_list=sscanf(temp_neuron_list_str,'%d-%d;'); +handles.temp_neuron_list=reshape(temp_neuron_list,2,[]); %two rows, first row-template, second row-the toAlign. +% Update handles structure +guidata(hObject, handles); + + +% --- Executes during object creation, after setting all properties. +function edit_del_CreateFcn(hObject, eventdata, handles) +% hObject handle to edit_del (see GCBO) +% eventdata reserved - to be defined in a future version of MATLAB +% handles empty - handles not created until after all CreateFcns called + +% Hint: edit controls usually have a white background on Windows. +% See ISPC and COMPUTER. +if ispc && isequal(get(hObject,'BackgroundColor'), get(0,'defaultUicontrolBackgroundColor')) + set(hObject,'BackgroundColor','white'); +end + + +% --- Executes on button press in pushbutton_del. +function pushbutton_del_Callback(hObject, eventdata, handles) +% hObject handle to pushbutton_del (see GCBO) +% eventdata reserved - to be defined in a future version of MATLAB +% handles structure with handles and user data (see GUIDATA) +whichpic=handles.temp_neuron_list(1); +whichneuron=handles.temp_neuron_list(2); +handles.M_intact=handles.M; +for i = 1:length(handles.M) + handles.M{i}{whichpic}(:,whichneuron)=[]; +end +display(['neuron ' num2str(whichneuron) 'in file ' num2str(whichpic) ' deleted.']) +set(handles.edit_del,'String',[]); +uicontrol(handles.slider1) +% Update handles structure +guidata(hObject, handles); +ToAlign_Callback(handles.ToAlign, eventdata, handles) + + +% --- Executes on button press in cancel_del. +function cancel_del_Callback(hObject, eventdata, handles) +% hObject handle to cancel_del (see GCBO) +% eventdata reserved - to be defined in a future version of MATLAB +% handles structure with handles and user data (see GUIDATA) + +handles.M=handles.M_intact; + +% Update handles structure +guidata(hObject, handles); +ToAlign_Callback(handles.ToAlign, eventdata, handles) diff --git a/BatchVerMO/motion/Precalculate_Minfo.m b/BatchVerMO/motion/Precalculate_Minfo.m new file mode 100644 index 0000000..22a53cc --- /dev/null +++ b/BatchVerMO/motion/Precalculate_Minfo.m @@ -0,0 +1,13 @@ +function handles=Precalculate_Minfo(M,handles) +diagr=tril(ones(numel(M),numel(M)),-1); +[row,col]=find(diagr); +handles.allpairs=[col,row]; + +% Opening look + +% % slider1 +set(handles.slider1,'Enable','off') +set(handles.slider1,'Min',1); +set(handles.slider1,'Max',size(handles.allpairs,1)); +set(handles.slider1,'Value',1); +set(handles.slider1,'SliderStep', [1/(size(handles.allpairs,1)-1), 1/(size(handles.allpairs,1)-1)]); \ No newline at end of file diff --git a/BatchVerMO/motion/cell2mat_ov.m b/BatchVerMO/motion/cell2mat_ov.m new file mode 100644 index 0000000..1d4feef --- /dev/null +++ b/BatchVerMO/motion/cell2mat_ov.m @@ -0,0 +1,33 @@ +function X = cell2mat_ov(I,xx_s,xx_f,yy_s,yy_f,zz_s,zz_f,overlap,sz) + +% converts a cell array to a matrix when the cell elements overlap +% INPUTS: +% I: cell array +% grid_size: true size of each element +% overlap: amount of overlap in each direction +% d1: number of rows of matrix +% d2: number of columns of matrix + +% OUTPUT: +% X: output matrix + +% Written by Eftychios A. Pnevmatikakis, Simons Foundation, 2016 +xx_s=xx_s-sz(1)+1; xx_f=xx_f-sz(1)+1; +yy_s=yy_s-sz(3)+1; yy_f=yy_f-sz(3)+1; +zz_s=zz_s-sz(5)+1; zz_f=zz_f-sz(5)+1; + + +sz=[sz(2)-sz(1)+1 sz(4)-sz(3)+1 sz(6)-sz(5)+1]; +X = NaN(sz(1),sz(2),sz(3)); + +if length(sz) == 2; sz(3) = 1; end + +for i = 1:length(xx_f) + for j = 1:length(yy_f) + for k = 1:length(zz_f) + extended_grid = [max(xx_s(i)-overlap(1),1),min(xx_f(i)+overlap(1),sz(1)),max(yy_s(j)-overlap(2),1),min(yy_f(j)+overlap(2),sz(2)),max(zz_s(k)-overlap(3),1),min(zz_f(k)+overlap(3),sz(3))]; + X(xx_s(i):xx_f(i),yy_s(j):yy_f(j),zz_s(k):zz_f(k)) = ... + I{i,j,k}(1+(xx_s(i)-extended_grid(1)):end-(extended_grid(2)-xx_f(i)),1+(yy_s(j)-extended_grid(3)):end-(extended_grid(4)-yy_f(j)),1+(zz_s(k)-extended_grid(5)):end-(extended_grid(6)-zz_f(k))); + end + end +end \ No newline at end of file diff --git a/BatchVerMO/motion/cell2mat_ov_sum.m b/BatchVerMO/motion/cell2mat_ov_sum.m new file mode 100644 index 0000000..9522f5c --- /dev/null +++ b/BatchVerMO/motion/cell2mat_ov_sum.m @@ -0,0 +1,48 @@ +function X = cell2mat_ov_sum(I,xx_s,xx_f,yy_s,yy_f,zz_s,zz_f,overlap,sz,Bs) + +% converts a cell array to a matrix when the cell elements overlap +% INPUTS: +% I: cell array +% grid_size: true size of each element +% overlap: amount of overlap in each direction +% d1: number of rows of matrix +% d2: number of columns of matrix + +% OUTPUT: +% X: output matrix + +% Written by Eftychios A. Pnevmatikakis, Simons Foundation, 2016 + +if nargin < 10 || isempty(Bs) + Bs = cellfun(@(x) ones(size(x)), I,'un',0); +end +%X = zeros([sz,size(I{1,1},length(sz)+1)]); +xx_s=xx_s-sz(1)+1; xx_f=xx_f-sz(1)+1; +yy_s=yy_s-sz(3)+1; yy_f=yy_f-sz(3)+1; +zz_s=zz_s-sz(5)+1; zz_f=zz_f-sz(5)+1; +sz=[sz(2)-sz(1)+1 sz(4)-sz(3)+1 sz(6)-sz(5)+1]; +X = NaN(sz(1),sz(2),sz(3)); + +B = zeros(size(X)); +if length(sz) == 2; sz(3) = 1; end + +for i = 1:length(xx_f) + for j = 1:length(yy_f) + for k = 1:length(zz_f) + extended_grid = [max(xx_s(i)-overlap(1),1),min(xx_f(i)+overlap(1),sz(1)),max(yy_s(j)-overlap(2),1),min(yy_f(j)+overlap(2),sz(2)),max(zz_s(k)-overlap(3),1),min(zz_f(k)+overlap(3),sz(3))]; + %W = construct_weights([xx_s(i),xx_f(i),yy_s(j),yy_f(j),zz_s(k),zz_f(k)],extended_grid)'; + Xtemp = zeros(size(I{i,j,k})); + Btemp = Xtemp; + ind = ~isnan(I{i,j,k}); + Xtemp(ind) = Bs{i,j,k}(ind).*I{i,j,k}(ind); + Btemp(ind) = Bs{i,j,k}(ind); + %X(extended_grid(1):extended_grid(2),extended_grid(3):extended_grid(4),extended_grid(5):extended_grid(6)) = X(extended_grid(1):extended_grid(2),extended_grid(3):extended_grid(4),extended_grid(5):extended_grid(6)) + Bs{i,j,k}.*I{i,j,k}; + %B(extended_grid(1):extended_grid(2),extended_grid(3):extended_grid(4),extended_grid(5):extended_grid(6)) = B(extended_grid(1):extended_grid(2),extended_grid(3):extended_grid(4),extended_grid(5):extended_grid(6)) + Bs{i,j,k}.*(I{i,j,k}~=0); + X(extended_grid(1):extended_grid(2),extended_grid(3):extended_grid(4),extended_grid(5):extended_grid(6)) = X(extended_grid(1):extended_grid(2),extended_grid(3):extended_grid(4),extended_grid(5):extended_grid(6)) + Xtemp; + B(extended_grid(1):extended_grid(2),extended_grid(3):extended_grid(4),extended_grid(5):extended_grid(6)) = B(extended_grid(1):extended_grid(2),extended_grid(3):extended_grid(4),extended_grid(5):extended_grid(6)) + Btemp; + end + end +end + +X = X./B; +%X(isnan(X))=0; \ No newline at end of file diff --git a/BatchVerMO/motion/construct_grid.m b/BatchVerMO/motion/construct_grid.m new file mode 100644 index 0000000..b45eb32 --- /dev/null +++ b/BatchVerMO/motion/construct_grid.m @@ -0,0 +1,45 @@ +function [xx_s,xx_f,yy_s,yy_f,zz_s,zz_f,xx_us,xx_uf,yy_us,yy_uf,zz_us,zz_uf] = construct_grid(grid_size,mot_uf,d1,d2,d3,min_patch_size,gridstartend) +if isempty(gridstartend) + start_xx=1; start_yy=1; start_zz=1; + end_xx=d1; end_yy=d2; end_zz=d3; +else + start_xx=gridstartend(1); end_xx=gridstartend(2); + start_yy=gridstartend(3); end_yy=gridstartend(4); + start_zz=gridstartend(5); end_zz=gridstartend(6); +end +xx_s = start_xx:grid_size(1):end_xx; +yy_s = start_yy:grid_size(2):end_yy; +zz_s = start_zz:grid_size(3):end_zz; + +xx_f = [xx_s(2:end)-1,end_xx]; +yy_f = [yy_s(2:end)-1,end_yy]; + +if length(zz_s)>1 + zz_f = [zz_s(2:end)-1,end_zz]; +else + zz_f=end_zz; +end + +if xx_f(end)-xx_s(end) + 1 < min_patch_size(1) && length(xx_s) > 1; xx_s(end) = []; xx_f(end-1) = []; end +if yy_f(end)-yy_s(end) + 1 < min_patch_size(2) && length(yy_s) > 1; yy_s(end) = []; yy_f(end-1) = []; end +if zz_f(end)-zz_s(end) + 1 < min_patch_size(3) && length(zz_s) > 1; zz_s(end) = []; zz_f(end-1) = []; end + +grid_size_us = floor(grid_size./mot_uf); +if mot_uf(1) > 1 + xx_us = start_xx:grid_size_us(1):end_xx; + xx_uf = [xx_us(2:end)-1,end_xx]; +else + xx_us = xx_s; xx_uf = xx_f; +end +if mot_uf(2) > 1 + yy_us = start_yy:grid_size_us(2):end_yy; + yy_uf = [yy_us(2:end)-1,end_yy]; +else + yy_us = yy_s; yy_uf = yy_f; +end +if mot_uf(3) > 1 + zz_us = start_zz:grid_size_us(3):end_zz; + zz_uf = [zz_us(2:end)-1,end_zz]; +else + zz_us = zz_s; zz_uf = zz_f; +end \ No newline at end of file diff --git a/BatchVerMO/motion/loadAs.m b/BatchVerMO/motion/loadAs.m new file mode 100644 index 0000000..0cd2774 --- /dev/null +++ b/BatchVerMO/motion/loadAs.m @@ -0,0 +1,23 @@ +%% Load data +DatadirForA=fullfile(cnmfedir,'*PartI_Afinal*'); +Alist=dir(DatadirForA); +AnumInFolder=numel(Alist); + +AsfromDaysPic=[]; +AsfromDaysCell=cell(1,AnumInFolder); +AsfromDaysCell_central=cell(1,AnumInFolder); +%sizes=[]; +for ia=1:AnumInFolder + Anext=load(fullfile(cnmfedir,Alist(ia).name)); + + Atemp=Anext.Afinal; + AsfromDaysCell{ia}=Atemp; % individual column is each A, for actual motion correction + + Atemp=centralA(Atemp); + AsfromDaysCell_central{ia}=Atemp; + + k = size(Atemp,2); + %sizes=[sizes k]; + + AsfromDaysPic=cat(3,AsfromDaysPic,A2image(Atemp,300,400)); % whole picture, for registering, getting shifts +end \ No newline at end of file diff --git a/BatchVerMO/motion/mat2cell_ov.m b/BatchVerMO/motion/mat2cell_ov.m new file mode 100644 index 0000000..4905dfb --- /dev/null +++ b/BatchVerMO/motion/mat2cell_ov.m @@ -0,0 +1,29 @@ +function I = mat2cell_ov(X,xx_s,xx_f,yy_s,yy_f,zz_s,zz_f,overlap,nd) + +% converts a matrix into a cell array with overlapping elements +% INPUTS: +% X: Input matrix +% grid_size: size of each element without overlap +% overlap: amount of overlap +% sz: spatial size of X + +% OUTPUT: +% I: output cell array + +% Written by Eftychios A. Pnevmatikakis, Simons Foundation, 2016 + +I = cell(length(xx_s),length(yy_s),length(zz_s)); +%nd = ndims(X); +%if nd == 2; sz(3) = 1; end +for i = 1:length(xx_s) + for j = 1:length(yy_s) + for k = 1:length(zz_s) + extended_grid = [max(xx_s(i)-overlap(1),xx_s(1)),min(xx_f(i)+overlap(1),xx_f(end)),max(yy_s(j)-overlap(2),yy_s(1)),min(yy_f(j)+overlap(2),yy_f(end)),max(zz_s(k)-overlap(3),zz_s(1)),min(zz_f(k)+overlap(3),zz_f(end))]; + if nd == 2 + I{i,j} = X(extended_grid(1):extended_grid(2),extended_grid(3):extended_grid(4),:); + else + I{i,j,k} = X(extended_grid(1):extended_grid(2),extended_grid(3):extended_grid(4),extended_grid(5):extended_grid(6),:); + end + end + end +end \ No newline at end of file diff --git a/BatchVerMO/motion/motioncorrection.m b/BatchVerMO/motion/motioncorrection.m new file mode 100644 index 0000000..6bc46dd --- /dev/null +++ b/BatchVerMO/motion/motioncorrection.m @@ -0,0 +1,81 @@ +function [M_final,xxsfyysf,T]=motioncorrection(AsfromDaysCell,AsfromDaysPic,options_nonrigid,gridstartend) +% T is the tranformation matrix +%% +Y=AsfromDaysPic; +clear AsfromDaysPic +AnumInFolder=numel(AsfromDaysCell); +sizes=cellfun(@(x) size(x,2),AsfromDaysCell); + +M=cell(1,AnumInFolder); M_central=M; +T=M; +ind_del=cell(1,AnumInFolder); + +% the only left out one; +M{1}{1}=AsfromDaysCell{1}; +M_central{1}{1}=centralA(AsfromDaysCell{1}); +T{1}{1}=zeros(size(Y,1),size(Y,2),2); +ind_del{1}{1}=false(1,sizes(1)); + + +for ia=2:AnumInFolder + shifts_up_1=cell(1,ia-1); shifts_up_2=cell(1,ia-1); + + siz_oneday=sizes(ia); + % The day's own data + M{ia}{ia}=AsfromDaysCell{ia}; + M_central{ia}{ia}=centralA(AsfromDaysCell{ia}); + T{ia}{ia}=zeros(size(Y,1),size(Y,2),2); + ind_del{ia}{ia}=false(1,siz_oneday); + % Previous days + M_buffer=[]; + for io=ia-1:-1:1 + if io==ia-1 + Y_template=Y(:,:,io); + else + Y_template=A2image(cat(2,M{io}{1:ia-1}),size(Y,1),size(Y,2)); + end + if io==ia-1 + Y_toregister=Y(:,:,ia); + else + Y_toregister=A2image(cat(2,M{io+1}{io+1},M{io+1}{ia}),size(Y,1),size(Y,2)); + end + + %tic; [M{ia},shifts{ia},~,xxsfyysf,ind_del{ia}] = normcorre_BatchVer(Y_ex_oneday,options_nonrigid,Y_oneday,siz_ex_oneday,As_ex_oneday,startendgrid,update_num); toc + tic; [M_temp,shifts,shifts_up_1{io},shifts_up_2{io},~,xxsfyysf,ind_del_temp] = ... + normcorre_BatchVer(Y_toregister,options_nonrigid,Y_template,siz_oneday,M{io+1}{ia},gridstartend); toc + M{io}{ia}=M_temp{1}; + M_central{io}{ia}=centralA(M_temp{1}); + ind_del{io}{ia}=ind_del_temp{1}; + + % inverse consec + shifts_up_1_tmp=sum(cat(3,shifts_up_1{io:ia-1}),3); shifts_up_2_tmp=sum(cat(3,shifts_up_2{io:ia-1}),3); + T=zeros(size(Y,1),size(Y,2),2); + T(gridstartend(1):gridstartend(2),gridstartend(3):gridstartend(4),1)=shifts_up_2_tmp; + T(gridstartend(1):gridstartend(2),gridstartend(3):gridstartend(4),2)=shifts_up_1_tmp; + T{io}{ia}=T; + T{ia}{io}=-T; + + Mf_temp=[]; + ind_del{ia}{io}=false(1,sizes(io)); + for ni=1:sizes(io) + Y_one_neuron=reshape(AsfromDaysCell{io}(:,ni),size(Y,1),size(Y,2)); + Y_one_neuron(gridstartend(1):gridstartend(2),gridstartend(3):gridstartend(4),gridstartend(5):gridstartend(6)) = ... + imwarp(Y_one_neuron(gridstartend(1):gridstartend(2),gridstartend(3):gridstartend(4),gridstartend(5):gridstartend(6)),-cat(3,shifts_up_2_tmp.*(-1),shifts_up_1_tmp.*(-1)),options_nonrigid.shifts_method); + if any(Y_one_neuron)==0 + ind_del{ia}{io}(ni)=true; + end + Mf_temp=[Mf_temp reshape(Y_one_neuron,[],1)]; + end + M{ia}{io}=Mf_temp; + M_central{ia}{io}=centralA(M{ia}{io}); + end +end + +%% +ind_del_full_cell=cellfun(@(x) cell2mat(x), ind_del, 'UniformOutput',0); +ind_del_full=sum(reshape(cell2mat(ind_del_full_cell),1,sum(sizes),[]),3)>0; %sum(cat(3,ind_del{:}))>0; +ind_del_full_cell=mat2cell(~ind_del_full,1,sizes); +N_eachday=cellfun(@(x) sum(x), ind_del_full_cell); +M_concat=cellfun(@(x) cat(2,x{:}), M, 'UniformOutput',0); +M_del = cellfun(@(x) x(:,~ind_del_full), M_concat, 'UniformOutput',0); +M_final = cellfun(@(x) mat2cell(x, [size(x,1)], N_eachday), M_del, 'UniformOutput',0); \ No newline at end of file diff --git a/BatchVerMO/motion/normcorre_batch.m b/BatchVerMO/motion/normcorre_batch.m new file mode 100644 index 0000000..82df8dc --- /dev/null +++ b/BatchVerMO/motion/normcorre_batch.m @@ -0,0 +1,450 @@ +function [M_final,shifts_g,template] = normcorre_batch(Y,options,template) + +% online motion correction through DFT subpixel registration +% Based on the dftregistration.m function from Manuel Guizar and Jim Fienup + +% INPUTS +% Y: Input data, can be already loaded in memory as a 3D +% tensor, a memory mapped file, or a pointer to a tiff stack +% options: options structure for motion correction (optional, rigid registration is performed if not provided) +% template: provide template (optional) + +% OUTPUTS +% M_final: motion corrected data +% shifts_up: upsampled shifts +% shifts: originally calculated shifts +% template: calculated template + +%% first determine filetype + +if isa(Y,'char') + [~,~,ext] = fileparts(Y); + ext = ext(2:end); + if strcmpi(ext,'tif') || strcmpi(ext,'tiff'); + tiffInfo = imfinfo(Y); + filetype = 'tif'; + T = length(tiffInfo); + sizY = [tiffInfo(1).Height,tiffInfo(1).Width,T]; + elseif strcmpi(ext,'mat') + filetype = 'mem'; + Y = matfile(Y,'Writable',true); + sizY = size(Y.Y); + T = sizY(end); + elseif strcmpi(ext,'hdf5') || strcmpi(ext,'h5'); + filetype = 'hdf5'; + fileinfo = hdf5info(Y); + data_name = fileinfo.GroupHierarchy.Datasets.Name; + sizY = fileinfo.GroupHierarchy.Datasets.Dims; + T = sizY(end); + elseif strcmpi(ext,'raw') + filetype = 'raw'; + fid = fopen(Y); + FOV = [options.d1,options.d2]; + bitsize = options.bitsize; + imsize = FOV(1)*FOV(2)*bitsize; % Bit size of single frame + current_seek = ftell(fid); + fseek(fid, 0, 1); + file_length = ftell(fid); + fseek(fid, current_seek, -1); + T = file_length/imsize; + sizY = [FOV,T]; + fclose(fid); + end +elseif isobject(Y); + filetype = 'mem'; + sizY = size(Y,'Y'); + T = sizY(end); +else % array loaded in memory + filetype = 'mat'; + %Y = double(Y); + sizY = size(Y); + T = sizY(end); +end + +nd = length(sizY)-1; % determine whether imaging is 2d or 3d +sizY = sizY(1:nd); +%% set default parameters if not present + +if ~exist('options','var') || isempty(options); + options = NoRMCorreSetParms('d1',sizY(1),'d2',sizY(2)); + if nd > 2; options.d3 = sizY(3); end +end + +memmap = options.memmap; +grid_size = options.grid_size; +mot_uf = options.mot_uf; +min_patch_size = options.min_patch_size; +overlap_pre = options.overlap_pre; +overlap_post = options.overlap_post; +upd_template = options.upd_template; +bin_width = options.bin_width; +buffer_width = options.buffer_width; +max_dev_g = options.max_dev; +init_batch = options.init_batch; +us_fac = options.us_fac; +method = options.method; +filename = options.mem_filename; +iter = options.iter; +add_value = options.add_value; +max_shift = options.max_shift; + +while mod(T,bin_width) == 1 + if T == 1 + error('Movie appears to have only one frame. Use the function normcorre instead') + end + bin_width = bin_width + 1; +end + +%% first check for offset due to bi-directional scanning + +if options.correct_bidir + col_shift = correct_bidirectional_offset(Y,options.nFrames,options.bidir_us); + if col_shift + if strcmpi(options.shifts_method,'fft') + options.shifts_method = 'cubic'; + fprintf('Offset %1.1f pixels due to bidirectional scanning detected. Cubic shifts will be applied. \n',col_shift); + end + end +else + col_shift = 0; +end + + +%% read initial batch and compute template + +init_batch = min(T,init_batch); +perm = randperm(T,init_batch); +switch filetype + case 'tif' + Y1 = imread(Y,'Index',perm(1),'Info',tiffInfo); + Y_temp = zeros(sizY(1),sizY(2),init_batch,'like',Y1); + Y_temp(:,:,1) = Y1; + for tt = 2:init_batch + Y_temp(:,:,tt) = imread(Y,'Index',perm(tt),'Info',tiffInfo); + end + case 'hdf5' + Y_temp = bigread2(Y,1,init_batch); + case 'mem' + if nd == 2; Y_temp = Y.Y(:,:,1:init_batch); elseif nd == 3; Y_temp = Y.Y(:,:,:,1:init_batch); end + case 'mat' + if nd == 2; Y_temp = Y(:,:,perm); elseif nd == 3; Y_temp = Y(:,:,:,perm); end + case 'raw' + Y_temp = read_raw_file(Y,1,init_batch,FOV,bitsize); +end +data_type = class(Y_temp); +Y_temp = single(Y_temp); + +if nargin < 3 || isempty(template) + template_in = median(Y_temp,nd+1)+add_value; +else + template_in = single(template + add_value); +end + +[d1,d2,d3,~] = size(Y_temp); +if nd == 2; d3 = 1; end +%% setup grids for patches + +[xx_s,xx_f,yy_s,yy_f,zz_s,zz_f,xx_us,xx_uf,yy_us,yy_uf,zz_us,zz_uf] = construct_grid(grid_size,mot_uf,d1,d2,d3,min_patch_size); +shifts_g = struct('shifts',cell(T,1),'shifts_up',cell(T,1),'diff',cell(T,1)); +temp_cell = mat2cell_ov(template_in,xx_us,xx_uf,yy_us,yy_uf,zz_us,zz_uf,overlap_post,sizY); + +%% precompute some quantities that are used repetitively for template matching and applying shifts +Nr = cell(size(temp_cell)); +Nc = cell(size(temp_cell)); +Np = cell(size(temp_cell)); +Bs = cell(size(temp_cell)); +for i = 1:length(xx_us) + for j = 1:length(yy_us) + for k = 1:length(zz_us) + [nr,nc,np] = size(temp_cell{i,j,k}); + nr = ifftshift(-fix(nr/2):ceil(nr/2)-1); + nc = ifftshift(-fix(nc/2):ceil(nc/2)-1); + np = ifftshift(-fix(np/2):ceil(np/2)-1); + [Nc{i,j,k},Nr{i,j,k},Np{i,j,k}] = meshgrid(nc,nr,np); + extended_grid = [max(xx_us(i)-overlap_post(1),1),min(xx_uf(i)+overlap_post(1),d1),max(yy_us(j)-overlap_post(2),1),min(yy_uf(j)+overlap_post(2),d2),max(zz_us(k)-overlap_post(3),1),min(zz_uf(k)+overlap_post(3),d3)]; + Bs{i,j,k} = permute(construct_weights([xx_us(i),xx_uf(i),yy_us(j),yy_uf(j),zz_us(k),zz_uf(k)],extended_grid),[2,1,3]); + end + end +end +if nd == 2; Np = cellfun(@(x) 0,Nr,'un',0); end + +%% +maxNumCompThreads(1); +template = mat2cell_ov(template_in,xx_s,xx_f,yy_s,yy_f,zz_s,zz_f,overlap_pre,sizY); +temp_mat = template_in; +use_windowing = options.use_windowing; +phase_flag = options.phase_flag; + +if use_windowing + fftTemp = cellfun(@fftn,cellfun(@han,template,'un',0),'un',0); + fftTempMat = fftn(han(temp_mat)); +else + fftTemp = cellfun(@fftn,template,'un',0); + fftTempMat = fftn(temp_mat); +end + +if nd == 2; buffer = mat2cell_ov(zeros(d1,d2,bin_width),xx_s,xx_f,yy_s,yy_f,zz_s,zz_f,overlap_pre,sizY); end +if nd == 3; buffer = mat2cell_ov(zeros(d1,d2,d3,bin_width),xx_s,xx_f,yy_s,yy_f,zz_s,zz_f,overlap_pre,sizY); end + +if ~strcmpi(options.output_type,'mat') + options.mem_batch_size = max(min(round(options.mem_batch_size/bin_width)*bin_width,T),1); + if nd == 2; mem_buffer = zeros(d1,d2,options.mem_batch_size,'single'); end + if nd == 3; mem_buffer = zeros(d1,d2,d3,options.mem_batch_size,'single'); end +end + +switch lower(options.output_type) + case 'mat' + M_final = zeros([sizY,T],data_type); + case 'memmap' + M_final = matfile(filename,'Writable',true); + if nd == 2; M_final.Y(d1,d2,T) = zeros(1,data_type); end + if nd == 3; M_final.Y(d1,d2,d3,T) = zeros(1,data_type); end + M_final.Yr(d1*d2*d3,T) = zeros(1,data_type); + case {'hdf5','h5'} + if exist(options.h5_filename,'file') + [pathstr,fname,ext] = fileparts(options.h5_filename); + new_filename = fullfile(pathstr,[fname,'_',datestr(now,30),ext]); + warning_msg = ['File ',options.h5_filename,'already exists. Saving motion corrected file as',new_filename]; + warning('%s',warning_msg); + options.h5_filename = new_filename; + end + M_final = options.h5_filename; + if nd == 2 + h5create(options.h5_filename,['/',options.h5_groupname],[d1,d2,Inf],'Chunksize',[d1,d2,options.mem_batch_size],'Datatype',data_type); + elseif nd == 3 + h5create(options.h5_filename,['/',options.h5_groupname],[d1,d2,d3,Inf],'Chunksize',[d1,d2,d3,options.mem_batch_size],'Datatype',data_type); + end + case {'tif','tiff'} + M_final = ['motion corrected file has been saved as ', options.tiff_filename]; + opts_tiff.append = true; + opts_tiff.big = true; + if nd == 3 + error('Saving volumetric tiff stacks is currently not supported. Use a different filetype'); + end + otherwise + error('This filetype is currently not supported') +end + +cnt_buf = 0; +fprintf('Template initialization complete. \n') +%% + + +for it = 1:iter + for t = 1:bin_width:T + switch filetype + case 'tif' + Ytm = zeros(sizY(1),sizY(2),min(t+bin_width-1,T)-t+1,'single'); + for tt = 1:min(t+bin_width-1,T)-t+1 + Ytm(:,:,tt) = single(imread(Y,'Index',t+tt-1,'Info',tiffInfo)); + end + case 'hdf5' + Ytm = single(h5read(Y,data_name,[ones(1,nd),t],[sizY(1:nd),min(t+bin_width-1,T)-t+1])); + case 'mem' + if nd == 2; Ytm = single(Y.Y(:,:,t:min(t+bin_width-1,T))); end + if nd == 3; Ytm = single(Y.Y(:,:,:,t:min(t+bin_width-1,T))); end + case 'mat' + if nd == 2; Ytm = single(Y(:,:,t:min(t+bin_width-1,T))); end + if nd == 3; Ytm = single(Y(:,:,:,t:min(t+bin_width-1,T))); end + case 'raw' + Ytm = single(read_raw_file(Y,t,min(t+bin_width-1,T)-t+1,FOV,bitsize)); + end + + if nd == 2; Ytc = mat2cell(Ytm,d1,d2,ones(1,size(Ytm,ndims(Ytm)))); end + if nd == 3; Ytc = mat2cell(Ytm,d1,d2,d3,ones(1,size(Ytm,ndims(Ytm)))); end + Mf = cell(size(Ytc)); + lY = length(Ytc); + %buffer = cell(length(xx_us),length(yy_us),length(zz_us),size(Ytm,ndims(Ytm))); + shifts = struct('shifts',cell(lY,1),'shifts_up',cell(lY,1),'diff',cell(lY,1)); + %buf = struct('Mf',cell(lY,1)); + parfor ii = 1:lY + Yt = Ytc{ii}; + minY = min(Yt(:)); + maxY = max(Yt(:)); + Yc = mat2cell_ov(Yt,xx_s,xx_f,yy_s,yy_f,zz_s,zz_f,overlap_pre,sizY); + if use_windowing + fftY = cellfun(@fftn, cellfun(@han,Yc, 'un',0),'un',0); + else + fftY = cellfun(@fftn, Yc, 'un',0); + end + + M_fin = cell(length(xx_us),length(yy_us),length(zz_us)); %zeros(size(Y_temp)); + shifts_temp = zeros(length(xx_s),length(yy_s),length(zz_s),nd); + diff_temp = zeros(length(xx_s),length(yy_s),length(zz_s)); + if numel(M_fin) > 1 + if use_windowing + if nd == 2; out_rig = dftregistration_min_max(fftTempMat,fftn(han(Yt)),us_fac,-max_shift,max_shift,phase_flag); lb = out_rig(3:4); ub = out_rig(3:4); end + if nd == 3; out_rig = dftregistration_min_max_3d(fftTempMat,fftn(han(Yt)),us_fac,-max_shift,max_shift,phase_flag); lb = out_rig(3:5); ub = out_rig(3:5); end + else + if nd == 2; out_rig = dftregistration_min_max(fftTempMat,fftn(Yt),us_fac,-max_shift,max_shift,phase_flag); lb = out_rig(3:4); ub = out_rig(3:4); end + if nd == 3; out_rig = dftregistration_min_max_3d(fftTempMat,fftn(Yt),us_fac,-max_shift,max_shift,phase_flag); lb = out_rig(3:5); ub = out_rig(3:5); end + end + max_dev = max_dev_g; + else + lb = -max_shift(1,nd); + ub = max_shift(1,nd); + max_dev = 0*max_dev_g; + end + for i = 1:length(xx_s) + for j = 1:length(yy_s) + for k = 1:length(zz_s) + if nd == 2 + %[output,Greg] = dftregistration_min_max(fftTemp{i,j,k},fftY{i,j,k},us_fac,lb-max_dev(1:2),ub+max_dev(1:2)); + output = dftregistration_min_max(fftTemp{i,j,k},fftY{i,j,k},us_fac,lb-max_dev(1:2),ub+max_dev(1:2),phase_flag); + elseif nd == 3 + output = dftregistration_min_max_3d(fftTemp{i,j,k},fftY{i,j,k},us_fac,lb-max_dev,ub+max_dev,phase_flag); + shifts_temp(i,j,k,3) = output(5); + end + + shifts_temp(i,j,k,1) = output(3); + shifts_temp(i,j,k,2) = output(4); + diff_temp(i,j,k) = output(2); + if all([length(xx_s),length(yy_s),length(zz_s)] == 1) + M_fin{i,j,k} = shift_reconstruct(Yt,shifts_temp(i,j,k,:),diff_temp(i,j,k),us_fac,Nr{i,j,k},Nc{i,j,k},Np{i,j,k},options.boundary,add_value); + end + end + end + end + + shifts(ii).shifts = shifts_temp; + shifts(ii).diff = diff_temp; + switch lower(options.shifts_method) + case 'fft' + if any([length(xx_s),length(yy_s),length(zz_s)] > 1) + if mot_uf(3) > 1 + tform = affine3d(diag([mot_uf(:);1])); + diff_up = imwarp(diff_temp,tform,'OutputView',imref3d([length(xx_uf),length(yy_uf),length(zz_uf)])); + shifts_up = zeros([size(diff_up),3]); + for dm = 1:3; shifts_up(:,:,:,dm) = imwarp(shifts_temp(:,:,:,dm),tform,'OutputView',imref3d([length(xx_uf),length(yy_uf),length(zz_uf)])); end + else + shifts_up = imresize(shifts_temp,[length(xx_uf),length(yy_uf)]); + diff_up = imresize(diff_temp,[length(xx_uf),length(yy_uf)]); + end + shifts(ii).shifts_up = shifts_up; + shifts(ii).diff = diff_up; + for i = 1:length(xx_uf) + for j = 1:length(yy_uf) + for k = 1:length(zz_uf) + extended_grid = [max(xx_us(i)-overlap_post(1),1),min(xx_uf(i)+overlap_post(1),d1),max(yy_us(j)-overlap_post(2),1),min(yy_uf(j)+overlap_post(2),d2),max(zz_us(k)-overlap_post(3),1),min(zz_uf(k)+overlap_post(3),d3)]; + I_temp = Yt(extended_grid(1):extended_grid(2),extended_grid(3):extended_grid(4),extended_grid(5):extended_grid(6)); + M_fin{i,j,k} = shift_reconstruct(I_temp,shifts_up(i,j,k,:),diff_up(i,j,k),us_fac,Nr{i,j,k},Nc{i,j,k},Np{i,j,k},options.boundary,add_value); + %M_fin{i,j,k} = shift_reconstruct2(I_temp,shifts_up(i,j,k,:),'bilinear',diff_up(i,j,k),us_fac,Nr{i,j,k},Nc{i,j,k},Np{i,j,k},options.boundary,add_value); + end + end + end + else + shifts_up = shifts_temp; + shifts(ii).shifts_up = shifts(ii).shifts; + end + gx = max(abs(reshape(diff(shifts_up,[],1),[],1))); + gy = max(abs(reshape(diff(shifts_up,[],2),[],1))); + gz = max(abs(reshape(diff(shifts_up,[],3),[],1))); + flag_interp = max([gx;gy;gz;0])<0.5; % detect possible smearing + + if flag_interp + Mf{ii} = cell2mat_ov_sum(M_fin,xx_us,xx_uf,yy_us,yy_uf,zz_us,zz_uf,overlap_post,sizY,Bs) - add_value; + else + Mf{ii} = cell2mat_ov(M_fin,xx_us,xx_uf,yy_us,yy_uf,zz_us,zz_uf,overlap_post,sizY) - add_value; + end + Mf{ii}(Mf{ii}maxY)=maxY; + + otherwise + shifts(ii).shifts_up = shifts(ii).shifts; + if nd == 3 + tform = affine3d(diag([mot_uf(:);1])); + shifts_up = zeros([options.d1,options.d2,options.d3,3]); + for dm = 1:3; shifts_up(:,:,:,dm) = imwarp(shifts_temp(:,:,:,dm),tform,'OutputView',imref3d([options.d1,options.d2,options.d3])); end + shifts_up(2:2:end,:,:,2) = shifts_up(2:2:end,:,:,2) + col_shift; + Mf{ii} = imwarp(Yt,-cat(3,shifts_up(:,:,2),shifts_up(:,:,1)),options.shifts_method); + else + shifts_up = imresize(shifts_temp,[options.d1,options.d2]); + shifts_up(2:2:end,:,2) = shifts_up(2:2:end,:,2) + col_shift; + Mf{ii} = imwarp(Yt,-cat(3,shifts_up(:,:,2),shifts_up(:,:,1)),options.shifts_method); + end + end + end + + shifts_g(t:min(t+bin_width-1,T)) = shifts; + Mf = cell2mat(Mf); + + if ~strcmpi(options.output_type,'mat') + rem_mem = rem(t+lY-1,options.mem_batch_size); + if rem_mem == 0; rem_mem = options.mem_batch_size; end + if nd == 2; mem_buffer(:,:,rem_mem-lY+1:rem_mem) = cast(Mf,data_type); end + if nd == 3; mem_buffer(:,:,:,rem_mem-lY+1:rem_mem) = cast(Mf,data_type); end + end + if it == iter + switch lower(options.output_type) + case 'mat' + if nd == 2; M_final(:,:,t:min(t+bin_width-1,T)) = cast(Mf,data_type); end + if nd == 3; M_final(:,:,:,t:min(t+bin_width-1,T)) = cast(Mf,data_type); end + case 'memmap' + if rem_mem == options.mem_batch_size || t+lY-1 == T + if nd == 2; M_final.Y(:,:,t+lY-rem_mem:t+lY-1) = mem_buffer(:,:,1:rem_mem); end + if nd == 3; M_final.Y(:,:,:,t+lY-rem_mem:t+lY-1) = mem_buffer(:,:,:,1:rem_mem); end + M_final.Yr(:,t+lY-rem_mem:t+lY-1) = reshape(mem_buffer(1:d1*d2*d3*rem_mem),d1*d2*d3,rem_mem); + end + case {'hdf5','h5'} + if rem_mem == options.mem_batch_size || t+lY-1 == T + if nd == 2; h5write(options.h5_filename,['/',options.h5_groupname],mem_buffer(:,:,1:rem_mem),[ones(1,nd),t+lY-rem_mem],[sizY(1:nd),rem_mem]); end + if nd == 3; h5write(options.h5_filename,['/',options.h5_groupname],mem_buffer(:,:,:,1:rem_mem),[ones(1,nd),t+lY-rem_mem],[sizY(1:nd),rem_mem]); end + end + case {'tif','tiff'} + if rem_mem == options.mem_batch_size || t+lY-1 == T + saveastiff(cast(mem_buffer(:,:,1:rem_mem),data_type),options.tiff_filename,opts_tiff); + end + end + end + + % update template + fprintf('%i out of %i frames registered, iteration %i out of %i \n',t+lY-1,T,it,iter) + if upd_template + cnt_buf = cnt_buf + 1; + if nd == 2; buffer = mat2cell_ov(Mf,xx_s,xx_f,yy_s,yy_f,zz_s,zz_f,overlap_pre,sizY); end + if nd == 3; buffer = mat2cell_ov(Mf,xx_s,xx_f,yy_s,yy_f,zz_s,zz_f,overlap_pre,sizY); end + if strcmpi(method{2},'mean') + new_temp = cellfun(@(x) nanmean(x,nd+1), buffer, 'UniformOutput',false); + elseif strcmpi(method{2},'median'); + new_temp = cellfun(@(x) nanmedian(x,nd+1), buffer, 'UniformOutput', false); + end + if any(reshape(cell2mat(cellfun(@(x) any(isnan(x(:))), new_temp, 'un',0)),[],1)) + parfor i = 1:numel(new_temp) + new_temp{i}(isnan(new_temp{i})) = template{i}(isnan(new_temp{i})); + end + end + if strcmpi(method{1},'mean') + cnt = t/bin_width + 1; + template = cellfun(@plus, cellfun(@(x) x*(cnt-1)/cnt, template,'un',0), cellfun(@(x) x*1/cnt, new_temp,'un',0), 'un',0); + elseif strcmpi(method{1},'median'); + if cnt_buf <= buffer_width + if nd == 2; buffer_med(:,:,cnt_buf) = cell2mat_ov(new_temp,xx_s,xx_f,yy_s,yy_f,zz_s,zz_f,overlap_pre,sizY); end + if nd == 3; buffer_med(:,:,:,cnt_buf) = cell2mat_ov(new_temp,xx_s,xx_f,yy_s,yy_f,zz_s,zz_f,overlap_pre,sizY); end + else + buffer_med = circshift(buffer_med,[zeros(1,nd),-1]); + if nd == 2; buffer_med(:,:,buffer_width) = cell2mat_ov(new_temp,xx_s,xx_f,yy_s,yy_f,zz_s,zz_f,overlap_pre,sizY); end + if nd == 3; buffer_med(:,:,:,buffer_width) = cell2mat_ov(new_temp,xx_s,xx_f,yy_s,yy_f,zz_s,zz_f,overlap_pre,sizY); end + end + template = mat2cell_ov(nanmedian(buffer_med,nd+1),xx_s,xx_f,yy_s,yy_f,zz_s,zz_f,overlap_pre,sizY); + end + temp_mat = cell2mat_ov(template,xx_s,xx_f,yy_s,yy_f,zz_s,zz_f,overlap_pre,sizY); + if use_windowing + fftTemp = cellfun(@fftn, cellfun(@han,template, 'un',0),'un',0); + fftTempMat = fftn(han(temp_mat)); + else + fftTemp = cellfun(@fftn, template, 'un',0); + fftTempMat = fftn(temp_mat); + end + end + end + + if it == iter + template = cellfun(@(x) x - add_value,template,'un',0); + template = cell2mat_ov(template,xx_s,xx_f,yy_s,yy_f,zz_s,zz_f,overlap_pre,sizY); + end + if memmap; + M_final.shifts = shifts_g; + M_final.template = template; + end + maxNumCompThreads(1); +end \ No newline at end of file diff --git a/BatchVerMO/motion/plot_grid.m b/BatchVerMO/motion/plot_grid.m new file mode 100644 index 0000000..dab03b8 --- /dev/null +++ b/BatchVerMO/motion/plot_grid.m @@ -0,0 +1,17 @@ +function [h1,h2]=plot_grid(xxsfyysf,overlap,d1,d2) + xx_s=xxsfyysf{1}; + xx_f=xxsfyysf{2}; + yy_s=xxsfyysf{3}; + yy_f=xxsfyysf{4}; + xx_s_e=[]; xx_f_e=[]; yy_s_e=[]; yy_f_e=[]; + for i=1:length(xx_s) + for j=1:length(yy_s) + xx_s_e=[xx_s_e max(xx_s(i)-overlap(1),xx_s(1))]; + xx_f_e=[xx_f_e min(xx_f(i)+overlap(1),xx_f(end))]; + yy_s_e=[yy_s_e max(yy_s(j)-overlap(2),yy_s(1))]; + yy_f_e=[yy_f_e min(yy_f(j)+overlap(2),yy_f(end))]; + end + end + + h1=plot([0 d2], [xx_s_e'; xx_f_e']*[1 1], ':', 'color', .7*[1 1 1]); + h2=plot([yy_s_e'; yy_f_e']*[1 1],[0 d1], ':', 'color', .7*[1 1 1]); diff --git a/BatchVerMO/normcorre_BatchVer.m b/BatchVerMO/normcorre_BatchVer.m new file mode 100644 index 0000000..8bd1742 --- /dev/null +++ b/BatchVerMO/normcorre_BatchVer.m @@ -0,0 +1,538 @@ +function [M_final,shifts_g,shifts_g_up_1,shifts_g_up_2,template,xxsfyysf,ind_del_full] = normcorre_BatchVer(Y,options,template,sizes,As,gridstartend,update_num) + +% online motion correction through DFT subpixel registration +% Based on the dftregistration.m function from Manuel Guizar and Jim Fienup + +% INPUTS +% Y: Input data, can be already loaded in memory as a 3D +% tensor, a memory mapped file, or a pointer to a tiff stack +% options: options structure for motion correction (optional, rigid registration is performed if not provided) +% template: provide template (optional) +% sizes: cell array, sizes{i}=size(Ai,2); + +% OUTPUTS +% M_final: motion corrected data +% shifts_up: upsampled shifts +% shifts: originally calculated shifts +% template: calculated template + +%% first determine filetype + +if isa(Y,'char') + [~,~,ext] = fileparts(Y); + ext = ext(2:end); + if strcmpi(ext,'tif') || strcmpi(ext,'tiff'); + tiffInfo = imfinfo(Y); + filetype = 'tif'; + T = length(tiffInfo); + sizY = [tiffInfo(1).Height,tiffInfo(1).Width,T]; + elseif strcmpi(ext,'mat') + filetype = 'mem'; + Y = matfile(Y,'Writable',true); + sizY = size(Y.Y); + T = sizY(end); + elseif strcmpi(ext,'hdf5') || strcmpi(ext,'h5'); + filetype = 'hdf5'; + fileinfo = hdf5info(Y); + data_name = fileinfo.GroupHierarchy.Datasets.Name; + sizY = fileinfo.GroupHierarchy.Datasets.Dims; + T = sizY(end); + elseif strcmpi(ext,'raw') + filetype = 'raw'; + fid = fopen(Y); + FOV = [options.d1,options.d2]; + bitsize = options.bitsize; + imsize = FOV(1)*FOV(2)*bitsize; % Bit size of single frame + current_seek = ftell(fid); + fseek(fid, 0, 1); + file_length = ftell(fid); + fseek(fid, current_seek, -1); + T = file_length/imsize; + sizY = [FOV,T]; + fclose(fid); + end +elseif isobject(Y); + filetype = 'mem'; + sizY = size(Y,'Y'); + T = sizY(end); +else % array loaded in memory + filetype = 'mat'; + %Y = double(Y); + sizY = size(Y); + T = sizY(end); +end + +nd = length(sizY)-1; % determine whether imaging is 2d or 3d +if or(nd==2,nd==3) + sizY = sizY(1:nd); +elseif nd==1 + T=1; +end +%% set default parameters if not present + +if ~exist('options','var') || isempty(options); + options = NoRMCorreSetParms('d1',sizY(1),'d2',sizY(2)); + if nd > 2; options.d3 = sizY(3); end +end + +memmap = options.memmap; +grid_size = options.grid_size; +mot_uf = options.mot_uf; +min_patch_size = options.min_patch_size; +overlap_pre = options.overlap_pre; +overlap_post = options.overlap_post; +upd_template = options.upd_template; +bin_width = options.bin_width; +buffer_width = options.buffer_width; +max_dev_g = options.max_dev; +init_batch = options.init_batch; +us_fac = options.us_fac; +method = options.method; +filename = options.mem_filename; +iter = options.iter; +add_value = options.add_value; +max_shift = options.max_shift; + +while mod(T,bin_width) == 1 + if T == 1 + error('Movie appears to have only one frame. Use the function normcorre instead') + end + bin_width = bin_width + 1; +end + +%% first check for offset due to bi-directional scanning + +if options.correct_bidir + col_shift = correct_bidirectional_offset(Y,options.nFrames,options.bidir_us); + if col_shift + if strcmpi(options.shifts_method,'fft') + options.shifts_method = 'cubic'; + fprintf('Offset %1.1f pixels due to bidirectional scanning detected. Cubic shifts will be applied. \n',col_shift); + end + end +else + col_shift = 0; +end + + +%% read initial batch and compute template + +init_batch = min(T,init_batch); +perm = randperm(T,init_batch); +switch filetype + case 'tif' + Y1 = imread(Y,'Index',perm(1),'Info',tiffInfo); + Y_temp = zeros(sizY(1),sizY(2),init_batch,'like',Y1); + Y_temp(:,:,1) = Y1; + for tt = 2:init_batch + Y_temp(:,:,tt) = imread(Y,'Index',perm(tt),'Info',tiffInfo); + end + case 'hdf5' + Y_temp = bigread2(Y,1,init_batch); + case 'mem' + if nd == 2; Y_temp = Y.Y(:,:,1:init_batch); elseif nd == 3; Y_temp = Y.Y(:,:,:,1:init_batch); end + case 'mat' + if nd == 2; Y_temp = Y(:,:,perm); elseif nd == 3; Y_temp = Y(:,:,:,perm); + elseif nd==1; Y_temp = Y(:,:); end + case 'raw' + Y_temp = read_raw_file(Y,1,init_batch,FOV,bitsize); +end +data_type = class(Y_temp); +Y_temp = single(Y_temp); + +if nargin < 3 || isempty(template) + template_in = median(Y_temp,nd+1)+add_value; +else + template_in = single(template + add_value); +end + +[d1,d2,d3,~] = size(Y_temp); +if or(nd==1,nd == 2); d3 = 1; end +%% setup grids for patches + +[xx_s,xx_f,yy_s,yy_f,zz_s,zz_f,xx_us,xx_uf,yy_us,yy_uf,zz_us,zz_uf] = construct_grid(grid_size,mot_uf,d1,d2,d3,min_patch_size,gridstartend); +shifts_g = struct('shifts',cell(T,1),'shifts_up',cell(T,1),'diff',cell(T,1)); +temp_cell = mat2cell_ov(template_in,xx_us,xx_uf,yy_us,yy_uf,zz_us,zz_uf,overlap_post,2); +xxsfyysf{1}=xx_s; xxsfyysf{2}=xx_f; xxsfyysf{3}=yy_s; xxsfyysf{4}=yy_f; +%% precompute some quantities that are used repetitively for template matching and applying shifts +Nr = cell(size(temp_cell)); +Nc = cell(size(temp_cell)); +Np = cell(size(temp_cell)); +Bs = cell(size(temp_cell)); +for i = 1:length(xx_us) + for j = 1:length(yy_us) + for k = 1:length(zz_us) + [nr,nc,np] = size(temp_cell{i,j,k}); + nr = ifftshift(-fix(nr/2):ceil(nr/2)-1); + nc = ifftshift(-fix(nc/2):ceil(nc/2)-1); + np = ifftshift(-fix(np/2):ceil(np/2)-1); + [Nc{i,j,k},Nr{i,j,k},Np{i,j,k}] = meshgrid(nc,nr,np); + extended_grid = [max(xx_us(i)-overlap_post(1),gridstartend(1)),min(xx_uf(i)+overlap_post(1),gridstartend(2)),max(yy_us(j)-overlap_post(2),gridstartend(3)),min(yy_uf(j)+overlap_post(2),gridstartend(4)),max(zz_us(k)-overlap_post(3),gridstartend(5)),min(zz_uf(k)+overlap_post(3),gridstartend(6))]; + Bs{i,j,k} = permute(construct_weights([xx_us(i),xx_uf(i),yy_us(j),yy_uf(j),zz_us(k),zz_uf(k)],extended_grid),[2,1,3]); + end + end +end +if or(nd == 2,nd==1); Np = cellfun(@(x) 0,Nr,'un',0); end + +%% +maxNumCompThreads(1); +template = mat2cell_ov(template_in,xx_s,xx_f,yy_s,yy_f,zz_s,zz_f,overlap_pre,2); +temp_mat = template_in(gridstartend(1):gridstartend(2),gridstartend(3):gridstartend(4),gridstartend(5):gridstartend(6)); +display('Originalline176') +display(size(temp_mat)) +use_windowing = options.use_windowing; +phase_flag = options.phase_flag; + +if use_windowing + fftTemp = cellfun(@fftn,cellfun(@han,template,'un',0),'un',0); + fftTempMat = fftn(han(temp_mat)); +else + fftTemp = cellfun(@fftn,template,'un',0); + fftTempMat = fftn(temp_mat); +end + +if nd==1||nd == 2; buffer = mat2cell_ov(zeros(d1,d2,bin_width),xx_s,xx_f,yy_s,yy_f,zz_s,zz_f,overlap_pre,2); end +if nd == 3; buffer = mat2cell_ov(zeros(d1,d2,d3,bin_width),xx_s,xx_f,yy_s,yy_f,zz_s,zz_f,overlap_pre,3); end + +if ~strcmpi(options.output_type,'mat') + options.mem_batch_size = max(min(round(options.mem_batch_size/bin_width)*bin_width,T),1); + if nd == 2; mem_buffer = zeros(d1,d2,options.mem_batch_size,'single'); end + if nd == 3; mem_buffer = zeros(d1,d2,d3,options.mem_batch_size,'single'); end +end + +switch lower(options.output_type) + case 'mat' + %M_final = zeros([sizY,T],data_type); + M_final = cell(1,T); + case 'memmap' + M_final = matfile(filename,'Writable',true); + if nd == 2; M_final.Y(d1,d2,T) = zeros(1,data_type); end + if nd == 3; M_final.Y(d1,d2,d3,T) = zeros(1,data_type); end + M_final.Yr(d1*d2*d3,T) = zeros(1,data_type); + case {'hdf5','h5'} + if exist(options.h5_filename,'file') + [pathstr,fname,ext] = fileparts(options.h5_filename); + new_filename = fullfile(pathstr,[fname,'_',datestr(now,30),ext]); + warning_msg = ['File ',options.h5_filename,'already exists. Saving motion corrected file as',new_filename]; + warning('%s',warning_msg); + options.h5_filename = new_filename; + end + M_final = options.h5_filename; + if nd == 2 + h5create(options.h5_filename,['/',options.h5_groupname],[d1,d2,Inf],'Chunksize',[d1,d2,options.mem_batch_size],'Datatype',data_type); + elseif nd == 3 + h5create(options.h5_filename,['/',options.h5_groupname],[d1,d2,d3,Inf],'Chunksize',[d1,d2,d3,options.mem_batch_size],'Datatype',data_type); + end + case {'tif','tiff'} + M_final = ['motion corrected file has been saved as ', options.tiff_filename]; + opts_tiff.append = true; + opts_tiff.big = true; + if nd == 3 + error('Saving volumetric tiff stacks is currently not supported. Use a different filetype'); + end + otherwise + error('This filetype is currently not supported') +end + +cnt_buf = 0; +fprintf('Template initialization complete. \n') +%% +ind_del_full = cell(1,T); +for it = 1:iter + for t = 1:bin_width:T + tpointer=t; + switch filetype + case 'tif' + Ytm = zeros(sizY(1),sizY(2),min(t+bin_width-1,T)-t+1,'single'); + for tt = 1:min(t+bin_width-1,T)-t+1 + Ytm(:,:,tt) = single(imread(Y,'Index',t+tt-1,'Info',tiffInfo)); + end + case 'hdf5' + Ytm = single(h5read(Y,data_name,[ones(1,nd),t],[sizY(1:nd),min(t+bin_width-1,T)-t+1])); + case 'mem' + if nd == 2; Ytm = single(Y.Y(:,:,t:min(t+bin_width-1,T))); end + if nd == 3; Ytm = single(Y.Y(:,:,:,t:min(t+bin_width-1,T))); end + case 'mat' + if nd == 1; Ytm = single(Y(:,:)); end + if nd == 2; Ytm = single(Y(:,:,t:min(t+bin_width-1,T))); end + if nd == 3; Ytm = single(Y(:,:,:,t:min(t+bin_width-1,T))); end + case 'raw' + Ytm = single(read_raw_file(Y,t,min(t+bin_width-1,T)-t+1,FOV,bitsize)); + end + + if nd == 2||nd == 1; + nd=2; + if size(Ytm,3)==1 + Ytc = mat2cell(Ytm,d1,d2); + else + Ytc = mat2cell(Ytm,d1,d2,ones(1,size(Ytm,ndims(Ytm)))); end + elseif nd == 3; + if size(Ytm,4)==1 + Ytc = mat2cell(Ytm,d1,d2,d3); + else + Ytc = mat2cell(Ytm,d1,d2,d3,ones(1,size(Ytm,ndims(Ytm)))); end + end + + Mf = cell(size(Ytc)); + lY = length(Ytc); + display('lY=') + display(lY) + ind_del = cell(1,lY); + %buffer = cell(length(xx_us),length(yy_us),length(zz_us),size(Ytm,ndims(Ytm))); + shifts = struct('shifts',cell(lY,1),'shifts_up',cell(lY,1),'diff',cell(lY,1)); + %buf = struct('Mf',cell(lY,1)); + parfor ii = 1:lY + display('line 281') + Yt = Ytc{ii}; + Yt_trunked = Yt(gridstartend(1):gridstartend(2),gridstartend(3):gridstartend(4),gridstartend(5):gridstartend(6)); + minY = min(Yt(:)); + maxY = max(Yt(:)); + Yc = mat2cell_ov(Yt,xx_s,xx_f,yy_s,yy_f,zz_s,zz_f,overlap_pre,2); + + if use_windowing + fftY = cellfun(@fftn, cellfun(@han,Yc, 'un',0),'un',0); + else + fftY = cellfun(@fftn, Yc, 'un',0); + end + + neuronnumber = sizes(ii+tpointer-1); + %M_fin = cell(length(sizes(ii)),length(xx_us),length(yy_us),length(zz_us)); %zeros(size(Y_temp)); + M_fin = cell(1,neuronnumber);%struct('M_fin',cell(length(xx_us),length(yy_us),length(zz_us))); + shifts_temp = zeros(length(xx_s),length(yy_s),length(zz_s),nd); + diff_temp = zeros(length(xx_s),length(yy_s),length(zz_s)); + display('line299') + if numel(shifts_temp) > 1 + if use_windowing + if nd == 2; out_rig = dftregistration_min_max(fftTempMat,fftn(han(Yt_trunked)),us_fac,-max_shift,max_shift,phase_flag); lb = out_rig(3:4); ub = out_rig(3:4); end + if nd == 3; out_rig = dftregistration_min_max_3d(fftTempMat,fftn(han(Yt_trunked)),us_fac,-max_shift,max_shift,phase_flag); lb = out_rig(3:5); ub = out_rig(3:5); end + else + if nd == 2; out_rig = dftregistration_min_max(fftTempMat,fftn(Yt_trunked),us_fac,-max_shift,max_shift,phase_flag); lb = out_rig(3:4); ub = out_rig(3:4); end + if nd == 3; out_rig = dftregistration_min_max_3d(fftTempMat,fftn(Yt_trunked),us_fac,-max_shift,max_shift,phase_flag); lb = out_rig(3:5); ub = out_rig(3:5); end + end + max_dev = max_dev_g; + else + lb = -max_shift(1,nd); + ub = max_shift(1,nd); + max_dev = 0*max_dev_g; + end + for i = 1:length(xx_s) + for j = 1:length(yy_s) + for k = 1:length(zz_s) + if nd == 2 + %[output,Greg] = dftregistration_min_max(fftTemp{i,j,k},fftY{i,j,k},us_fac,lb-max_dev(1:2),ub+max_dev(1:2)); + output = dftregistration_min_max(fftTemp{i,j,k},fftY{i,j,k},us_fac,lb-max_dev(1:2),ub+max_dev(1:2),phase_flag); + elseif nd == 3 + output = dftregistration_min_max_3d(fftTemp{i,j,k},fftY{i,j,k},us_fac,lb-max_dev,ub+max_dev,phase_flag); + shifts_temp(i,j,k,3) = output(5); + end + + shifts_temp(i,j,k,1) = output(3); + shifts_temp(i,j,k,2) = output(4); + diff_temp(i,j,k) = output(2); + if all([length(xx_s),length(yy_s),length(zz_s)] == 1) && strcmpi(options.shifts_method,'fft') + for ni=1:sizes(ii+tpointer-1) + Y_one_neuron=reshape(As(:,ni),d1,d2); + Y_one_neuron_c = mat2cell_ov(Y_one_neuron,xx_s,xx_f,yy_s,yy_f,zz_s,zz_f,overlap_pre,2); + M_fin{ni}{i,j,k} = shift_reconstruct(Y_one_neuron_c{i,j,k},shifts_temp(i,j,k,:),diff_temp(i,j,k),us_fac,Nr{i,j,k},Nc{i,j,k},Np{i,j,k},options.boundary,add_value); + end + end + end + end + end + + shifts(ii).shifts = shifts_temp; + shifts(ii).diff = diff_temp; + display('line340') + switch lower(options.shifts_method) + case 'fft' + display('Using fft.') + if any([length(xx_s),length(yy_s),length(zz_s)] > 1) + if mot_uf(3) > 1 + tform = affine3d(diag([mot_uf(:);1])); + diff_up = imwarp(diff_temp,tform,'OutputView',imref3d([length(xx_uf),length(yy_uf),length(zz_uf)])); + shifts_up = zeros([size(diff_up),3]); + for dm = 1:3; shifts_up(:,:,:,dm) = imwarp(shifts_temp(:,:,:,dm),tform,'OutputView',imref3d([length(xx_uf),length(yy_uf),length(zz_uf)])); end + else + shifts_up = imresize(shifts_temp,[length(xx_uf),length(yy_uf)]); + diff_up = imresize(diff_temp,[length(xx_uf),length(yy_uf)]); + end + shifts(ii).shifts_up = shifts_up; + shifts(ii).diff = diff_up; + for ni=1:sizes(ii+tpointer-1) + for i = 1:length(xx_uf) + for j = 1:length(yy_uf) + for k = 1:length(zz_uf) + Y_one_neuron=reshape(As(:,ni),d1,d2); + extended_grid_2 = [max(xx_us(i)-overlap_post(1),gridstartend(1)),min(xx_uf(i)+overlap_post(1),gridstartend(2)),max(yy_us(j)-overlap_post(2),gridstartend(3)),min(yy_uf(j)+overlap_post(2),gridstartend(4)),max(zz_us(k)-overlap_post(3),gridstartend(5)),min(zz_uf(k)+overlap_post(3),gridstartend(6))]; + I_temp = Y_one_neuron(extended_grid_2(1):extended_grid_2(2),extended_grid_2(3):extended_grid_2(4),extended_grid_2(5):extended_grid_2(6)); + %I_temp = Y_one_neuron(gridstartend(1):gridstartend(2),gridstartend(3):gridstartend(4),gridstartend(5):gridstartend(6)); + M_fin{ni}{i,j,k} = shift_reconstruct(I_temp,shifts_up(i,j,k,:),diff_up(i,j,k),us_fac,Nr{i,j,k},Nc{i,j,k},Np{i,j,k},options.boundary,add_value); + %M_fin{i,j,k} = shift_reconstruct2(I_temp,shifts_up(i,j,k,:),'bilinear',diff_up(i,j,k),us_fac,Nr{i,j,k},Nc{i,j,k},Np{i,j,k},options.boundary,add_value); + end + end + end + end + else + shifts_up = shifts_temp; + shifts(ii).shifts_up = shifts(ii).shifts; + end + gx = max(abs(reshape(diff(shifts_up,[],1),[],1))); + gy = max(abs(reshape(diff(shifts_up,[],2),[],1))); + gz = max(abs(reshape(diff(shifts_up,[],3),[],1))); + flag_interp = max([gx;gy;gz;0])<0.5; % detect possible smearing + + %Mf_each=cell(1,sizes(ii)); + Mf_temp=[]; + ind_del{ii} = false(1,sizes(ii+tpointer-1)); + for ni=1:sizes(ii+tpointer-1) + M_fin_tmp=M_fin{ni}; + if flag_interp + Mf_each = cell2mat_ov_sum(M_fin_tmp,xx_us,xx_uf,yy_us,yy_uf,zz_us,zz_uf,overlap_post,gridstartend,Bs) - add_value; + else + Mf_each = cell2mat_ov(M_fin_tmp,xx_us,xx_uf,yy_us,yy_uf,zz_us,zz_uf,overlap_post,gridstartend) - add_value; + end + Mf_each(Mf_eachmaxY)=maxY; + Y_one_neuron=reshape(As(:,ni),d1,d2); + Y_one_neuron(gridstartend(1):gridstartend(2),gridstartend(3):gridstartend(4),gridstartend(5):gridstartend(6))=Mf_each; + if any(Y_one_neuron)==0 + ind_del{ii}(ni)=true; + end + Mf_temp=[Mf_temp reshape(Y_one_neuron,[],1)]; + end + Mf{ii}=Mf_temp; + otherwise + display('Shift using other methods.') + %shifts(ii).shifts_up = shifts(ii).shifts; + if nd == 3 + tform = affine3d(diag([mot_uf(:);1])); + shifts_up = zeros([options.d1,options.d2,options.d3,3]); + for dm = 1:3; shifts_up(:,:,:,dm) = imwarp(shifts_temp(:,:,:,dm),tform,'OutputView',imref3d([options.d1,options.d2,options.d3])); end + shifts_up(2:2:end,:,:,2) = shifts_up(2:2:end,:,:,2) + col_shift; + Mf{ii} = imwarp(Yt,-cat(3,shifts_up(:,:,2),shifts_up(:,:,1)),options.shifts_method); + else + display('line410') + shifts_up = imresize(shifts_temp,[gridstartend(2)-gridstartend(1)+1,gridstartend(4)-gridstartend(3)+1]); + %shifts(ii).shifts_up = shifts_up; + shifts_up(2:2:end,:,2) = shifts_up(2:2:end,:,2) + col_shift; + shifts(ii).shifts_up = shifts_up; + Mf_temp=[]; + display('line416') + ind_del{ii} = false(1,sizes(ii+tpointer-1)); + for ni=1:sizes(ii+tpointer-1) + Y_one_neuron=reshape(As(:,ni),d1,d2); + Y_one_neuron(gridstartend(1):gridstartend(2),gridstartend(3):gridstartend(4),gridstartend(5):gridstartend(6)) = imwarp(Y_one_neuron(gridstartend(1):gridstartend(2),gridstartend(3):gridstartend(4),gridstartend(5):gridstartend(6)),-cat(3,shifts_up(:,:,2),shifts_up(:,:,1)),options.shifts_method); + if any(Y_one_neuron)==0 + ind_del{ii}(ni)=true; + end + Mf_temp=[Mf_temp reshape(Y_one_neuron,[],1)]; + end + display('line424') + Mf{ii}=Mf_temp; + end + end + end + + shifts_g(t:min(t+bin_width-1,T)) = shifts; + %shifts_g_up_1=zeros(size(shifts(1).shifts_up,1),size(shifts(1).shifts_up,2),T); shifts_g_up_2=zeros(size(shifts(1).shifts_up,1),size(shifts(1).shifts_up,2),T); + shifts_g_up_1(:,:) = shifts(1).shifts_up(:,:,1); + shifts_g_up_2(:,:) = shifts(1).shifts_up(:,:,2); + %Mf = cell2mat(Mf); + %%%%%%%%%%% Updating datasets%%%%%%%%%%%% + MfMAT=[]; + for ii=1:numel(Mf) + k = size(Mf{ii},2); C=ones(k,1); + Brainbow = Mf{ii}*C; Brainbow = reshape(Brainbow,300,400); + MfMAT=cat(3,MfMAT,Brainbow); + end + %Y(:,:,t:min(t+bin_width-1,T))=MfMAT; + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + + if ~strcmpi(options.output_type,'mat') + rem_mem = rem(t+lY-1,options.mem_batch_size); + if rem_mem == 0; rem_mem = options.mem_batch_size; end + if nd == 2; mem_buffer(:,:,rem_mem-lY+1:rem_mem) = cast(Mf,data_type); end + if nd == 3; mem_buffer(:,:,:,rem_mem-lY+1:rem_mem) = cast(Mf,data_type); end + end + if it == iter + switch lower(options.output_type) + case 'mat' + if nd == 2; M_final(t:min(t+bin_width-1,T)) = Mf; + ind_del_full(t:min(t+bin_width-1,T)) = ind_del; end %cast(Mf,data_type); end + if nd == 3; M_final(:,:,:,t:min(t+bin_width-1,T)) = cast(Mf,data_type); end + case 'memmap' + if rem_mem == options.mem_batch_size || t+lY-1 == T + if nd == 2; M_final.Y(:,:,t+lY-rem_mem:t+lY-1) = mem_buffer(:,:,1:rem_mem); end + if nd == 3; M_final.Y(:,:,:,t+lY-rem_mem:t+lY-1) = mem_buffer(:,:,:,1:rem_mem); end + M_final.Yr(:,t+lY-rem_mem:t+lY-1) = reshape(mem_buffer(1:d1*d2*d3*rem_mem),d1*d2*d3,rem_mem); + end + case {'hdf5','h5'} + if rem_mem == options.mem_batch_size || t+lY-1 == T + if nd == 2; h5write(options.h5_filename,['/',options.h5_groupname],mem_buffer(:,:,1:rem_mem),[ones(1,nd),t+lY-rem_mem],[sizY(1:nd),rem_mem]); end + if nd == 3; h5write(options.h5_filename,['/',options.h5_groupname],mem_buffer(:,:,:,1:rem_mem),[ones(1,nd),t+lY-rem_mem],[sizY(1:nd),rem_mem]); end + end + case {'tif','tiff'} + if rem_mem == options.mem_batch_size || t+lY-1 == T + saveastiff(cast(mem_buffer(:,:,1:rem_mem),data_type),options.tiff_filename,opts_tiff); + end + end + end + + % update template + fprintf('%i out of %i frames registered, iteration %i out of %i \n',t+lY-1,T,it,iter) + if upd_template %and(upd_template,t<=update_num) + display('Here') + display(t) + display('Updating Template Here') + cnt_buf = cnt_buf + 1; + [~,~,~,~,zz_s_temp,zz_f_temp,~,~,~,~,~,~] = construct_grid([grid_size(1) grid_size(2) size(MfMAT,3)],mot_uf,d1,d2,size(MfMAT,3), min_patch_size,[gridstartend(1:5), size(MfMAT,3)]); + buffer = mat2cell_ov(MfMAT,xx_s,xx_f,yy_s,yy_f,zz_s_temp,zz_f_temp,overlap_pre,3); + if strcmpi(method{2},'mean') + new_temp = cellfun(@(x) nanmean(x,nd+1), buffer, 'UniformOutput',false); + elseif strcmpi(method{2},'median'); + new_temp = cellfun(@(x) nanmedian(x,nd+1), buffer, 'UniformOutput', false); + end + if any(reshape(cell2mat(cellfun(@(x) any(isnan(x(:))), new_temp, 'un',0)),[],1)) + parfor i = 1:numel(new_temp) + new_temp{i}(isnan(new_temp{i})) = template{i}(isnan(new_temp{i})); + end + end + if strcmpi(method{1},'mean') + cnt = t/bin_width + 1; + template = cellfun(@plus, cellfun(@(x) x*(cnt-1)/cnt, template,'un',0), cellfun(@(x) x*1/cnt, new_temp,'un',0), 'un',0); + elseif strcmpi(method{1},'median'); + if cnt_buf <= buffer_width + if nd == 2; buffer_med(:,:,cnt_buf) = cell2mat_ov(new_temp,xx_s,xx_f,yy_s,yy_f,zz_s,zz_f,overlap_pre,gridstartend); end + if nd == 3; buffer_med(:,:,:,cnt_buf) = cell2mat_ov(new_temp,xx_s,xx_f,yy_s,yy_f,zz_s,zz_f,overlap_pre,gridstartend); end + else + buffer_med = circshift(buffer_med,[zeros(1,nd),-1]); + if nd == 2; buffer_med(:,:,buffer_width) = cell2mat_ov(new_temp,xx_s,xx_f,yy_s,yy_f,zz_s,zz_f,overlap_pre,gridstartend); end + if nd == 3; buffer_med(:,:,:,buffer_width) = cell2mat_ov(new_temp,xx_s,xx_f,yy_s,yy_f,zz_s,zz_f,overlap_pre,gridstartend); end + end + template = mat2cell_ov(nanmedian(buffer_med,nd+1),xx_s-gridstartend(1)+1,xx_f-gridstartend(1)+1,yy_s-gridstartend(3)+1,yy_f-gridstartend(3)+1,zz_s-gridstartend(5)+1,zz_f-gridstartend(5)+1,overlap_pre,2); + end + temp_mat = cell2mat_ov(template,xx_s,xx_f,yy_s,yy_f,zz_s,zz_f,overlap_pre,gridstartend); + + if use_windowing + fftTemp = cellfun(@fftn, cellfun(@han,template, 'un',0),'un',0); + fftTempMat = fftn(han(temp_mat)); + else + fftTemp = cellfun(@fftn, template, 'un',0); + fftTempMat = fftn(temp_mat); + end + display(size(fftTempMat)) + end + + end + + if it == iter + template = cellfun(@(x) x - add_value,template,'un',0); + template = cell2mat_ov(template,xx_s,xx_f,yy_s,yy_f,zz_s,zz_f,overlap_pre,gridstartend); + + end + if memmap; + M_final.shifts = shifts_g; + M_final.template = template; + end + maxNumCompThreads(1); +end \ No newline at end of file diff --git a/BatchVerMO/test.m b/BatchVerMO/test.m new file mode 100644 index 0000000..0e2a4dd --- /dev/null +++ b/BatchVerMO/test.m @@ -0,0 +1,4 @@ +function test(Y) +global y x +y=Y; +x=y; \ No newline at end of file diff --git a/BatchVerMO/threshold_by_diff.m b/BatchVerMO/threshold_by_diff.m new file mode 100644 index 0000000..08b8397 --- /dev/null +++ b/BatchVerMO/threshold_by_diff.m @@ -0,0 +1,25 @@ +function threshold_value=threshold_by_diff(P,resolution,number) +% threshold is where consecutive diff smaller than 'resolution' for 'number' pixels +% threshold_value returned in the end is the actual value of P at the +% threshold. + +% Shijie Gu, Fee Lab at McGovern, MIT-BCS; ShanghaiTech University. + +if isempty(number) + number=10; +end + +PNR0_sorted=sort(P,'descend'); +PNR0_diff=[2; diff(PNR0_sorted)]'; %2 is a random number +[L_low, numRegions_low] = bwlabel(abs(PNR0_diff)resolution); +sizes_high=[];for i=1:numRegions_high; sizes_high=[sizes_high,sum(L_high==i)]; end +if find(sizes_low>number)~=1 + threshold=sum(sizes_high(1:find(sizes_low>number)-1))+sum(sizes_low(1:find(sizes_low>number)-1)); + threshold_value=PNR0_sorted(threshold); +else + threshold_value=PNR0_sorted(1); + warning('Please lower the ''resolution'' or increase ''number''.') +end + diff --git a/CNMFE_First.m b/CNMFE_First.m new file mode 100644 index 0000000..09c7110 --- /dev/null +++ b/CNMFE_First.m @@ -0,0 +1,16 @@ +%% First Part (3h) +codeDir='/home/shijiegu/CNMF_E_New'; +display('Dir done') +addpath(genpath(codeDir)); +path2data='/home/shijiegu/feevault/ProcessedCalciumData/6938_FirstTutNewSyll/compiled.mat'; +path2out='/om/user/shijiegu/analysisOut_April05/'; + +% no input +CNMFE_Read_In(path2data,path2out) + +% input needed +CNMFE_Basic_Parameters(15,25,false,'ar1','thresholded',true,true,true); + +global Ysiz +patches = construct_patches(Ysiz(1:2),[64 64],[25 25],[16 16]); +[RESULTS] = CNMFE_Load_PatchY(patches, 1, 15676,1, 1); \ No newline at end of file diff --git a/HandlingParforbyGalen/init_par_rng.m b/HandlingParforbyGalen/init_par_rng.m new file mode 100644 index 0000000..8f575a1 --- /dev/null +++ b/HandlingParforbyGalen/init_par_rng.m @@ -0,0 +1,24 @@ +function init_par_rng(seedNo, varargin) +persistent p; +if isempty(p) + p = inputParser(); + addParameter(p, 'method', 'simdTwister'); +end +parse(p, varargin{:}); +Opt = p.Results; + +rng(seedNo, Opt.method); +%% Create workers if needed +poolObj = gcp(); +poolSize = poolObj.NumWorkers; +badSeed = true; +while badSeed + randSeeds = randi(intmax, 1, poolSize); + seedDiffs = pairwisediffs(randSeeds); + badSeed = any(seedDiffs == 0); +end +randOffsets = distributed(randSeeds); +spmd + rng(getLocalPart(randOffsets), Opt.method); +end +end \ No newline at end of file diff --git a/HandlingParforbyGalen/maybe_spawn_workers.m b/HandlingParforbyGalen/maybe_spawn_workers.m new file mode 100644 index 0000000..3c11672 --- /dev/null +++ b/HandlingParforbyGalen/maybe_spawn_workers.m @@ -0,0 +1,28 @@ +function [poolObj, ownPool, poolSize] = maybe_spawn_workers(nWorkersStrOrNumeric) +% written by Galen Lynch, fixes issue with parfor on openmind. +%% Create workers if needed +poolObj = gcp('nocreate'); +if isempty(poolObj) + %% Check to see if numWorkers needs to be converted from string to double + if ischar(nWorkersStrOrNumeric) + nWorkers = str2double(nWorkersStrOrNumeric); + else + nWorkers = nWorkersStrOrNumeric; + end + if nWorkers > 1 + ownPool = true; + c = parcluster(); + t = tempname(); + mkdir(t); + c.JobStorageLocation = t; + poolObj = parpool(c, nWorkers); + poolSize = poolObj.NumWorkers; + else + ownPool = false; + poolSize = 0; + end +else + ownPool = false; + poolSize = poolObj.NumWorkers; +end +end \ No newline at end of file diff --git a/HandlingParforbyGalen/pairwisediffs.m b/HandlingParforbyGalen/pairwisediffs.m new file mode 100644 index 0000000..71214c4 --- /dev/null +++ b/HandlingParforbyGalen/pairwisediffs.m @@ -0,0 +1,43 @@ +function diffMatOrCell = pairwisediffs(ptA, ptB) +%PAIRWISEDIFFS finds all pairwise differences between two matrices +% For two matrices with the same number of rows, PAIRWISEDIFFS finds +% the distance between all pairs of points from matrix A and matrix B. +% +% [diffMat] = PAIRWISEDIFFS(ptA) returns the difference between each +% pair of points a row of the matrix ptA, excluding the difference +% between each point and itself. +% +% [diffMat} = PAIRWISEDIFFS(ptA, ptB) returns the differences +% between every point in a row of ptA and all the other points in the +% corresponding row of ptB. ptA and ptB must have the same number of +% rows, or one must only have one row. +% +% Written by Galen Lynch 10/04/2014 +% email: glynch@mit.edu +nRepA = size(ptA, 1); +nPtA = size(ptA, 2); +if ~exist('ptB', 'var') %Auto + assert(~iscell(ptA), 'expecting a matrix'); + if nPtA < 2 + diffMatOrCell = nan(nRepA, 0); + return + end + nDiffs = nchoosek(nPtA, 2); + diffMatOrCell = nan(nRepA, nDiffs); + pos = 1; + for ptNo = 1:(nPtA - 1) + thisNDiff = nPtA - ptNo; + diffMatOrCell(:, pos:(pos+ thisNDiff - 1)) = ... + bsxfun(@minus, ptA(:, (ptNo + 1):end), ptA(:, ptNo)); + pos = pos+ thisNDiff; + end + diffMatOrCell = abs(diffMatOrCell); +else + nRepB = size(ptB, 1); + hasSingleton = nRepB == 1 || nRepA == 1; + assert(nRepA == nRepB || hasSingleton,... + 'matrices must have same size in the first dimension'); + diffMatOrCell = reshape(bsxfun(@minus, ptA,... + permute(ptB, [1, 3, 2])), nRepA, []); +end +end \ No newline at end of file diff --git a/ca_source_extraction/@Sources2D/Sources2D.m b/ca_source_extraction/@Sources2D/Sources2D.m index 915805b..6dd32d0 100644 --- a/ca_source_extraction/@Sources2D/Sources2D.m +++ b/ca_source_extraction/@Sources2D/Sources2D.m @@ -63,7 +63,7 @@ obj.options = CNMFSetParms(); obj.P =struct('mat_file', [], 'mat_data', [], 'indicator', '', 'k_options', 0, ... 'k_snapshot', 0, 'k_del', 0, 'k_merge', 0, 'k_trim', 0, 'sn', [], ... - 'kernel_pars',[], 'k_ids', 0); + 'kernel_pars',[], 'k_ids', 0,'THRESH',[]); if nargin>0 obj.options = CNMFSetParms(obj.options, varargin{:}); end @@ -441,7 +441,7 @@ function updateSpatial_nnls(obj, Y) end %% update temporal components for endoscope data - updateSpatial_endoscope(obj, Y, numIter, method, IND_thresh); + updateSpatial_endoscope(obj, Y, numIter, method, IND_thresh, allow_deletion); %% update temporal components function updateTemporal(obj, Y) @@ -450,7 +450,7 @@ function updateTemporal(obj, Y) end %% udpate temporal components with fast deconvolution - updateTemporal_endoscope(obj, Y, allow_deletion) + [C_offset,ind_del]= updateTemporal_endoscope(obj, Y, allow_deletion) updateTemporal_endoscope_parallel(obj, Y, smin) %% update temporal components without background @@ -778,7 +778,13 @@ function delete(obj, ind) obj.A(:, ind) = []; obj.C(ind, :) = []; - if ~isempty(obj.S) + + THRESH=obj.P.THRESH; + THRESH.Corr(ind)=[]; + + THRESH.PNR(ind)=[]; + if ~isempty(obj.S); + try obj.S(ind, :) = []; catch; end end if ~isempty(obj.C_raw) @@ -919,10 +925,12 @@ function exportAVI(obj, Y, min_max, avi_nm, col_map) end %% trim spatial components - function [ind_small] = trimSpatial(obj, thr, sz) + function [ind_small] = trimSpatial(obj, thr, sz, allow_deletion) % remove small nonzero pixels + if nargin<2; thr = 0.01; end if nargin<3; sz = 5; end + se = strel('square', sz); ind_small = false(size(obj.A, 2), 1); @@ -940,8 +948,10 @@ function exportAVI(obj, Y, min_max, avi_nm, col_map) end obj.A(:, m) = ai(:); end + % ind_small = find(ind_small); % obj.delete(ind_small); + end %% keep spatial shapes compact @@ -1641,9 +1651,13 @@ function playAC(obj, avi_file, cell_id, indt) obj.C = [obj.C; neuron.C]; obj.C_raw = [obj.C_raw; neuron.C_raw]; if obj.options.deconv_flag - obj.S = [obj.S; neuron.S]; obj.P.kernel_pars = [obj.P.kernel_pars; neuron.P.kernel_pars]; + obj.S = [obj.S; neuron.S]; end + obj.P.THRESH.Corr= [obj.P.THRESH.Corr neuron.P.THRESH.Corr]; + obj.P.THRESH.CorrOut= [obj.P.THRESH.CorrOut neuron.P.THRESH.CorrOut]; + obj.P.THRESH.PNR= [obj.P.THRESH.PNR neuron.P.THRESH.PNR]; + obj.P.THRESH.PNROut= [obj.P.THRESH.PNROut neuron.P.THRESH.PNROut]; end %% post process spatial component @@ -2023,8 +2037,11 @@ function correlation_pnr_batch(obj) end end - - %% manually draw ROI and show the mean fluorescence traces within the ROI + + %% New method added by Shijie Gu, since Jun,2017 + drawPNRCn(obj,min_pnr,min_corr) + + %% manually draw ROI and show the mean fluorescence traces within the ROI function y = drawROI(obj, Y, img, type) Y = obj.reshape(Y,1); if ~exist('img', 'var') || isempty(img) diff --git a/ca_source_extraction/@Sources2D/drawPNRCn.m b/ca_source_extraction/@Sources2D/drawPNRCn.m new file mode 100644 index 0000000..3504a40 --- /dev/null +++ b/ca_source_extraction/@Sources2D/drawPNRCn.m @@ -0,0 +1,26 @@ +function drawPNRCn(obj,min_pnr,min_corr) +global Picname outputdirDetails + +THRESH=obj.P.THRESH; +numberOfNeuron=length(THRESH.Corr); + +x = linspace(0,1); +y = min_pnr*min_corr./x; +figure +scatter(THRESH.CorrOut,THRESH.PNROut,20,'blue') +hold on +scatter(THRESH.Corr,THRESH.PNR,20,'red') +plot(x,y,'k') +plot([min_corr min_corr],[0 max(max(THRESH.PNROut),max(THRESH.PNR))],'k') +plot([0 1],[min_pnr min_pnr],'k') +MAXPNR=max(max(THRESH.PNROut),max(THRESH.PNR)); +axis([0 1 0 MAXPNR]) +text(THRESH.Corr,THRESH.PNR,cellstr(num2str((1:numberOfNeuron)'))','Color','black') +title(strcat('Current PNR and Corr cutoff: ',num2str(numberOfNeuron),'neurons'),'interpreter','none'); +xlabel('Corr'); +ylabel(strcat('PNR=',num2str(min_pnr))); +legend('Those outside neurons','Neuron seeds','Corr*PnrThresh','CorrThresh','PnrThresh') + +fignam=[outputdirDetails,Picname,strcat('PNR=',num2str(min_pnr)),'.png']; +saveas(gcf,fignam); +close(gcf); \ No newline at end of file diff --git a/ca_source_extraction/@Sources2D/initComponents_endoscope.m b/ca_source_extraction/@Sources2D/initComponents_endoscope.m index 16bfb04..ec23f39 100644 --- a/ca_source_extraction/@Sources2D/initComponents_endoscope.m +++ b/ca_source_extraction/@Sources2D/initComponents_endoscope.m @@ -13,7 +13,7 @@ % Cn: correlation image % PNR: peak to noise ratio %% Author: Pengcheng Zhou, Carnegie Mellon University, zhoupc1988@gmail.com - +% modified by Shijie Gu %% process parameters d1 = obj.options.d1; d2 = obj.options.d2; @@ -66,6 +66,7 @@ obj.S = zeros(size(obj.C)); end obj.Cn = Cn; + obj.P.THRESH=results.THRESH; return; elseif isscalar(patch_sz) % patch size has been assigned diff --git a/ca_source_extraction/@Sources2D/initComponents_endoscope2.m b/ca_source_extraction/@Sources2D/initComponents_endoscope2.m new file mode 100644 index 0000000..8cd84cb --- /dev/null +++ b/ca_source_extraction/@Sources2D/initComponents_endoscope2.m @@ -0,0 +1,200 @@ +function [center, Cn, PNR] = initComponents_endoscope2(obj, Y, K, patch_sz, debug_on, save_avi) +%% initializing spatial/temporal components for miceoendoscopic data +%% input: +% Y: d1 X d2 X T matrix or (d1*d2)X T matrix, video data +% K: scalar, maximum number of neurons +% patch_sz: scalar or vector, if its scalar, then the patch size is +% patch_sz * patch_sz; if its vector, then the optical field is divided +% into patch_sz(1) X patch_sz(2) patches. +% debug_on: options for showing procedure of detecting neurons +% save_avi: save the video of initialization procedure +%% Output: +% center: d*2 matrix, centers of all initialized neurons. +% Cn: correlation image +% PNR: peak to noise ratio +%% Author: Pengcheng Zhou, Carnegie Mellon University, zhoupc1988@gmail.com + +%% process parameters +d1 = obj.options.d1; +d2 = obj.options.d2; + +if isfield(obj.options, 'nk') % number of knots for creating spline basis + nk = obj.options.nk; +else + nk = 1; +end +% maximum neuron number in each patch +if (~exist('K', 'var')) || (isempty(K)) + % if K is not specified, use a very large number as default + K = round((d1*d2)); +end + +% exporting initialization procedures as a video +if ~exist('save_avi', 'var')||isempty(save_avi) + save_avi = false; +elseif ischar(save_avi) || (save_avi==true) + debug_on = true; % turn on debug mode +else + save_avi = false; %don't save initialization procedure +end + +% debug mode and exporting results +if ~exist('debug_on', 'var') + debug_on = false; +end + +options = obj.options; +% options.kernel = .kernel; +% divide the optical fields into multiple patches +if (~exist('patch_sz', 'var'))||(isempty(patch_sz))||(max(patch_sz(:))==1) + % use the whole optical field directly + if nk>1 % detrend data + Ydt = detrend_data(obj.reshape(double(Y), 1), nk); % detrend data + [results, center, Cn, PNR] = greedyROI_endoscope2(Ydt, K, options, debug_on, save_avi); + else % without detrending + [results, center, Cn, PNR] = greedyROI_endoscope2(Y, K, options, debug_on, save_avi); + end + obj.A = results.Ain; + obj.C = results.Cin; + obj.C_raw = results.Cin_raw; + if options.deconv_flag + obj.S = results.Sin; + obj.P.kernel_pars = results.kernel_pars; + else + obj.S = zeros(size(obj.C)); + end + obj.Cn = Cn; + return; +elseif isscalar(patch_sz) + % patch size has been assigned + r0_patch = (1:patch_sz:d1); % break points in the y axis + r0_patch(end) = d1; % break points in the x axis + c0_patch = (1:patch_sz:d2); + c0_patch(end) = d2; +elseif length(patch_sz)==2 + % the patch number has been specified. + r0_patch = round(linspace(1, d1, 1+patch_sz(1))); + c0_patch = round(linspace(1, d2, 1+patch_sz(2))); +else + fprintf('wrong parameters for patch_sz\n\n'); + return; +end + +%% start initialization +nr_patch = length(r0_patch)-1; % number of patches in y axis +nc_patch = length(c0_patch)-1; % number of patches in x axis +Y = obj.reshape(Y, 2); % represent each frame with a matrix +if ~isfield(options, 'bd') || isempty(options.bd') + options.bd = options.gSiz; % boundary pixesl to be ignored during the process of detecting seed pixels +end +bd = options.bd; +Ain = cell(nr_patch, nc_patch); % save spatial components of neurons in each patch +Cin = cell(nr_patch, nc_patch); % save temporal components of neurons in each patch, denoised trace +Sin = cell(nr_patch, nc_patch); % save temporal components of neurons in each patch, deconvolved trace +Cin_raw = cell(nr_patch, nc_patch); % save temporal components of neurons in each patch, raw trace +kernel_pars = cell(nr_patch, nc_patch); % save temporal components of neurons in each patch +center = cell(nr_patch, nc_patch); % save centers of all initialized neurons +Cn = zeros(d1, d2); +PNR = zeros(d1, d2); + +% initialize A and C for each patch +flag_patch = false(nr_patch, nc_patch); +for mr = 1:nr_patch + r0 = max(1, r0_patch(mr)-bd); % minimum row index of the patch + r1 = min(d1, r0_patch(mr+1)+bd-1); % maximum row index of the patch + for mc = 1:nc_patch + c0 = max(1, c0_patch(mc)-bd); % minimum column index of the patch + c1 = min(d2, c0_patch(mc+1)+bd-1); % maximum column index of the patch + tmp_options = options; + tmp_options.d1 = (r1-r0)+1; + tmp_options.d2 = (c1-c0)+1; + tmp_options.kernel = obj.kernel; + + % name of the video + if ischar(save_avi) + tmp_save_avi = sprintf('%s_%d_%d_%d_%d.avi', save_avi, r0, c0, r1, c1); + else + tmp_save_avi = save_avi; + end + + % take the patch from the raw data + nrows = (r1-r0+1); % number of rows in the patch + ncols = (c1-c0+1); %number of columns in the patch + Ypatch = double(reshape(Y(r0:r1, c0:c1, :), nrows*ncols, [])); + + % top patch, some signals have been explained by neurons in the top + % patch + if mr>1 + tmpA = reshape(Ain{mr-1, mc}, d1, d2, []); + tmpC = Cin{mr-1, mc}; + Ypatch = Ypatch - reshape(tmpA(r0:r1, c0:c1,:), nrows*ncols, [])*tmpC; + end + + % left patch, some signals have been explained by neurons in the + % left patch + if mc>1 + tmpA = reshape(Ain{mr, mc-1}, d1, d2, []); + tmpC = Cin{mr, mc-1}; + Ypatch = Ypatch-reshape(tmpA(r0:r1, c0:c1,:), nrows*ncols, [])*tmpC; + end + if nk>1 + Ypatch_dt = detrend_data(Ypatch, nk); % detrend data + [tmp_results, tmp_center, tmp_Cn, tmp_PNR, save_avi] = greedyROI_endoscope2(Ypatch_dt, K, tmp_options, debug_on, tmp_save_avi); + else + [tmp_results, tmp_center, tmp_Cn, tmp_PNR, save_avi] = greedyROI_endoscope2(Ypatch, K, tmp_options, debug_on, tmp_save_avi); + end + close(gcf); + tmp_Ain = tmp_results.Ain; + tmp_Cin = tmp_results.Cin; + tmp_Sin = tmp_results.Sin; + tmp_Cin_raw = tmp_results.Cin_raw; + tmp_kernel_pars = tmp_results.kernel_pars; + tmp_K = size(tmp_Ain, 2); % number of neurons within the selected patch + temp = zeros(d1, d2, tmp_K); % spatial components of all neurosn + temp(r0:r1, c0:c1, :) = reshape(full(tmp_Ain), tmp_options.d1, tmp_options.d2, []); + Ain{mr, mc} = reshape(temp, d1*d2, tmp_K); + Cin{mr, mc} = tmp_Cin; % temporal components of all neurons + Cin_raw{mr, mc} = tmp_Cin_raw; + if tmp_options.deconv_flag + Sin{mr, mc} = tmp_Sin; + kernel_pars{mr,mc} = tmp_kernel_pars; + end + center{mr, mc} = bsxfun(@plus, tmp_center, [r0-1, c0-1]); % centers + Cn(r0:r1, c0:c1) = max(Cn(r0:r1, c0:c1), tmp_Cn); + PNR(r0:r1, c0:c1) = max(PNR(r0:r1, c0:c1), tmp_PNR); + + % display initialization progress + flag_patch(mr, mc) = true; + clc; + fprintf('\n Progress......\n'); + for tmpm = 1:nr_patch + for tmpn=1:nc_patch + if flag_patch(tmpm, tmpn) + fprintf('X\t'); + else + fprintf('O\t'); + end + end + fprintf('\n'); + end + end +end + +%% export the results +Ain = cell2mat(reshape(Ain, 1, [])); +Cin = cell2mat(reshape(Cin, [], 1)); +Cin_raw = cell2mat(reshape(Cin_raw, [], 1)); +center = cell2mat(reshape(center, [], 1)); +obj.A = Ain; +obj.C = Cin; +obj.C_raw = Cin_raw; +if tmp_options.deconv_flag + Sin = cell2mat(reshape(Sin, [], 1)); + kernel_pars = cell2mat(reshape(kernel_pars, [], 1)); + obj.S = Sin; + obj.P.kernel_pars = kernel_pars; +else + obj.S = zeros(size(obj.C)); +end +obj.Cn = Cn; +end \ No newline at end of file diff --git a/ca_source_extraction/@Sources2D/quickMerge.m b/ca_source_extraction/@Sources2D/quickMerge.m index 99a2129..db06c00 100644 --- a/ca_source_extraction/@Sources2D/quickMerge.m +++ b/ca_source_extraction/@Sources2D/quickMerge.m @@ -44,6 +44,7 @@ S = obj.S; if isempty(S) || (size(S, 1)~=size(obj.C, 1)) + try S = diff(obj.C_raw, 1, 2); catch @@ -51,6 +52,7 @@ end S(:, end+1) = 0; + S(bsxfun(@lt, S, 2*get_noise_fft(S))) = 0; end S_corr = corr(S') - eye(K); diff --git a/ca_source_extraction/@Sources2D/updateSpatial_endoscope.m b/ca_source_extraction/@Sources2D/updateSpatial_endoscope.m index 78adb26..c67a37f 100644 --- a/ca_source_extraction/@Sources2D/updateSpatial_endoscope.m +++ b/ca_source_extraction/@Sources2D/updateSpatial_endoscope.m @@ -1,4 +1,4 @@ -function updateSpatial_endoscope(obj, Y, num, method) +function updateSpatial_endoscope(obj, Y, num, method, IND_thresh, allow_deletion) %% udpate spatial components %% inputs: @@ -8,6 +8,7 @@ function updateSpatial_endoscope(obj, Y, num, method) % overlapping at one pixel % method: method for updating the spatial components {'hals', 'nnls'}. % default: 'nnls' +% allow_deletion: true or false. %% Author: Pengcheng Zhou, Carnegie Mellon University. @@ -22,7 +23,9 @@ function updateSpatial_endoscope(obj, Y, num, method) num = 10; end end - +if ~exist('allow_deletion', 'var') + allow_deletion = true; +end %% determine the search locations search_method = obj.options.search_method; params = obj.options; @@ -32,16 +35,16 @@ function updateSpatial_endoscope(obj, Y, num, method) IND = logical(determine_search_location(obj.A, search_method, params)); %% estimate the noise -if and(strcmpi(method, 'hals_thresh') || strcmpi(method, 'nnls_thresh'), isempty(obj.P.sn)) - %% estimate the noise for all pixels - b0 =zeros(size(obj.A,1), 1); - sn = b0; - parfor m=1:size(obj.A,1) - [b0(m), sn(m)] = estimate_baseline_noise(Y(m, :)); - end - Y = bsxfun(@minus, Y, b0); - obj.P.sn = sn; -end +% if and(strcmpi(method, 'hals_thresh') || strcmpi(method, 'nnls_thresh'), isempty(obj.P.sn)) +% %% estimate the noise for all pixels +% b0 =zeros(size(obj.A,1), 1); +% sn = b0; +% parfor m=1:size(obj.A,1) +% [b0(m), sn(m)] = estimate_baseline_noise(Y(m, :)); +% end +% Y = bsxfun(@minus, Y, b0); +% obj.P.sn = sn; +% end %% update spatial components if strcmpi(method, 'hals') @@ -59,7 +62,9 @@ function updateSpatial_endoscope(obj, Y, num, method) obj.A = nnls_spatial(Y, obj.A, obj.C, IND, num); end +if allow_deletion %% thresholding the minimum number of neurons -obj.delete(sum(obj.A>0, 1)<=obj.options.min_pixel); + obj.delete(sum(obj.A>0, 1)<=obj.options.min_pixel); +end end diff --git a/ca_source_extraction/@Sources2D/updateTemporal_endoscope.m b/ca_source_extraction/@Sources2D/updateTemporal_endoscope.m index a1e6ff3..38af98d 100644 --- a/ca_source_extraction/@Sources2D/updateTemporal_endoscope.m +++ b/ca_source_extraction/@Sources2D/updateTemporal_endoscope.m @@ -1,4 +1,4 @@ -function [C_offset] = updateTemporal_endoscope(obj, Y, allow_deletion) +function [C_offset,ind_del] = updateTemporal_endoscope(obj, Y, allow_deletion) %% run HALS by fixating all spatial components % input: % Y: d*T, fluorescence data @@ -9,7 +9,12 @@ % Author: Pengcheng Zhou, Carnegie Mellon University, adapted from Johannes % options -maxIter = obj.options.maxIter; +global mode nam +if strcmp(mode,'initiation') + maxIter = obj.options.maxIter; +else + maxIter=1; +end deconv_options_0 = obj.options.deconv_options; if ~exist('allow_deletion', 'var') @@ -55,21 +60,43 @@ tmp_sn = sn_hist; b = b_hist; end - + temp = temp -b; sn(k) = tmp_sn; - % deconvolution if obj.options.deconv_flag - [ck, sk, deconv_options]= deconvolveCa(temp, deconv_options_0, 'maxIter', 2, 'sn', tmp_sn); - smin(k) = deconv_options.smin; - kernel_pars{k} = reshape(deconv_options.pars, 1, []); + if strcmp(mode,'initiation') + try + [ck, sk, deconv_options]= deconvolveCa(temp, deconv_options_0, 'sn', tmp_sn, 'maxIter', 2); + smin(k) = deconv_options.smin; + kernel_pars{k} = reshape(single(deconv_options.pars), 1, []); + catch %if not deconvolved successfully + ck = max(0, temp); + sk=zeros(1,size(C,2)); + if or(strcmp(deconv_options_0.type,'ar1'),strcmp(deconv_options_0.type,'kernel')) + kernel_pars{k}=single(0); + else + kernel_pars{k}=single(zeros(1,2)); + end + ind_del(k) = true; + end + end + + if strcmp(mode,'massive') + ck = temp;%max(0, temp); + sk=zeros(1,size(C,2)); + if or(strcmp(deconv_options_0.type,'ar1'),strcmp(deconv_options_0.type,'kernel')) + kernel_pars{k}=single(0); + else + kernel_pars{k}=single(zeros(1,2)); + end + ind_del(k) = true; + end temp = temp - deconv_options.b; else ck = max(0, temp); - end - % save convolution kernels and deconvolution results - C(k, :) = ck; + end + C(k, :) = ck; % save convolution kernels and deconvolution results (or not deconvolutioned) if sum(ck(2:end))==0 ind_del(k) = true; @@ -80,16 +107,22 @@ S(k, :) = sk; end C_raw(k, :) = temp; + end end end + obj.A = bsxfun(@times, A, sn); obj.C = bsxfun(@times, C, 1./sn'); obj.C_raw = bsxfun(@times, C_raw, 1./sn'); obj.S = bsxfun(@times, S, 1./sn'); -obj.P.kernel_pars =cell2mat( kernel_pars); +obj.P.kernel_pars =cell2mat(kernel_pars); obj.P.smin = smin/sn; obj.P.sn_neuron = sn; -if allow_deletion - obj.delete(ind_del); + +if strcmp(mode,'initiation') + if allow_deletion + obj.delete(ind_del); + end end + diff --git a/ca_source_extraction/@Sources2D/viewNeurons.m b/ca_source_extraction/@Sources2D/viewNeurons.m index 4f5942b..e680a87 100644 --- a/ca_source_extraction/@Sources2D/viewNeurons.m +++ b/ca_source_extraction/@Sources2D/viewNeurons.m @@ -92,8 +92,7 @@ if ~isempty(C2) plot(t, C2(ind(m), :)*max(obj.A(:, ind(m))), 'linewidth', 2); hold on; plot(t, obj.C(ind(m), :)*max(obj.A(:, ind(m))), 'r'); - else - + else plot(t, obj.C(ind(m), :)*max(obj.A(:, ind(m)))); end xlim([t(1), t(end)]); diff --git a/ca_source_extraction/demo_endoscope.m b/ca_source_extraction/demo_endoscope.m new file mode 100644 index 0000000..8bd4042 --- /dev/null +++ b/ca_source_extraction/demo_endoscope.m @@ -0,0 +1,234 @@ +%% New Version May2017 +% Fee Lab +%% clear workspace +clear; clc; close all; +global d1 d2 numFrame ssub tsub sframe num2read Fs neuron neuron_ds ... + neuron_full Ybg_weights; %#ok % global variables, don't change them manually + +addpath(genpath('\\feevault\shared\EmilyShijieShared\CNMF_E-master')) + +%% select data and map it to the RAM +nam = '\\feevault\shared\EmilyShijieShared\TestDatasets\CaELM_row3289.mat'; +cnmfe_choose_data; + +%% create Source2D class object for storing results and parameters +Fs = 6; % frame rate +ssub = 1; % spatial downsampling factor +tsub = 1; % temporal downsampling factor +gSig = 15; % width of the gaussian kernel, which can approximates the average neuron shape +gSiz = 25; % maximum diameter of neurons in the image plane. larger values are preferred. +neuron_full = Sources2D('d1',d1,'d2',d2, ... % dimensions of datasets + 'ssub', ssub, 'tsub', tsub, ... % downsampleing + 'gSig', gSig,... % sigma of the 2D gaussian that approximates cell bodies + 'gSiz', gSiz); % average neuron size (diameter) +neuron_full.Fs = Fs; % frame rate + +% with dendrites or not +with_dendrites = false; +if with_dendrites + % determine the search locations by dilating the current neuron shapes + neuron_full.options.search_method = 'dilate'; + neuron_full.options.bSiz = 20; +else + % determine the search locations by selecting a round area + neuron_full.options.search_method = 'ellipse'; + neuron_full.options.dist = 5; +end + +%% options for running deconvolution +neuron_full.options.deconv_options = struct('type', 'ar1', ... % model of the calcium traces. {'ar1', 'ar2'} + 'method', 'thresholded', ... % method for running deconvolution {'foopsi', 'constrained', 'thresholded'} + 'optimize_pars', true, ... % optimize AR coefficients + 'optimize_b', false, ... % optimize the baseline + 'optimize_smin', true); % optimize the threshold + +%% downsample data for fast and better initialization +sframe=1; % user input: first frame to read (optional, default:1) +num2read= numFrame; % user input: how many frames to read (optional, default: until the end) + +tic; +cnmfe_load_data; +fprintf('Time cost in downsapling data: %.2f seconds\n', toc); + +Y = neuron.reshape(Y, 1); % convert a 3D video into a 2D matrix + +%% compute correlation image and peak-to-noise ratio image. +cnmfe_show_corr_pnr; % this step is not necessary, but it can give you some... + % hints on parameter selection, e.g., min_corr & min_pnr + +%% initialization of A, C +% parameters +debug_on = false; % visualize the initialization procedue. +save_avi = false; %save the initialization procedure as an avi movie. +patch_par = [1,1]*1; %1; % divide the optical field into m X n patches and do initialization patch by patch. It can be used when the data is too large +K = []; % maximum number of neurons to search within each patch. you can use [] to search the number automatically + +min_corr = 0.75; % minimum local correlation for a seeding pixel +min_pnr = 9; % minimum peak-to-noise ratio for a seeding pixel +min_pixel = 4; % minimum number of nonzero pixels for each neuron +bd = 1; % number of rows/columns to be ignored in the boundary (mainly for motion corrected data) +neuron.updateParams('min_corr', min_corr, 'min_pnr', min_pnr, ... + 'min_pixel', min_pixel, 'bd', bd); +neuron.options.nk = 1; % number of knots for detrending + +% greedy method for initialization +tic; +[center, Cn, pnr] = neuron.initComponents_endoscope(Y, K, patch_par, debug_on, save_avi); +fprintf('Time cost in initializing neurons: %.2f seconds\n', toc); + +% show results +figure; +imagesc(Cn, [0.1, 0.95]); +hold on; plot(center(:, 2), center(:, 1), 'or'); +colormap; axis off tight equal; + +% sort neurons +[~, srt] = sort(max(neuron.C, [], 2), 'descend'); +neuron.orderROIs(srt); +neuron_init = neuron.copy(); + +%% iteratively update A, C and B +% parameters, merge neurons +display_merge = false; % visually check the merged neurons +view_neurons = false; % view all neurons + +% parameters, estimate the background +spatial_ds_factor = 1; % spatial downsampling factor. it's for faster estimation +thresh = 10; % threshold for detecting frames with large cellular activity. (mean of neighbors' activity + thresh*sn) + +bg_neuron_ratio = 1; % spatial range / diameter of neurons + +% parameters, estimate the spatial components +update_spatial_method = 'hals'; % the method for updating spatial components {'hals', 'hals_thresh', 'nnls', 'lars'} +Nspatial = 5; % this variable has different meanings: + %1) udpate_spatial_method=='hals' or 'hals_thresh', + %then Nspatial is the maximum iteration + %2) update_spatial_method== 'nnls', it is the maximum + %number of neurons overlapping at one pixel + +% parameters for running iteratiosn +nC = size(neuron.C, 1); % number of neurons + +maxIter = 2; % maximum number of iterations +miter = 1; +while miter <= maxIter + %% merge neurons, order neurons and delete some low quality neurons + if miter ==1 + merge_thr = [1e-1, 0.8, .1]; % thresholds for merging neurons + % corresponding to {sptial overlaps, temporal correlation of C, + %temporal correlation of S} + else + merge_thr = [0.6, 0.5, 0.1]; + end + % merge neurons + cnmfe_quick_merge; % run neuron merges + + %% udpate background (cell 1, the following three blocks can be run iteratively) + % estimate the background + tic; + cnmfe_update_BG; + fprintf('Time cost in estimating the background: %.2f seconds\n', toc); + % neuron.playMovie(Ysignal); % play the video data after subtracting the background components. + + %% update spatial & temporal components + tic; + for m=1:2 + %temporal + neuron.updateTemporal_endoscope(Ysignal); + cnmfe_quick_merge; % run neuron merges + %spatial + neuron.updateSpatial_endoscope(Ysignal, Nspatial, update_spatial_method); + neuron.trimSpatial(0.01, 3); % for each neuron, apply imopen first and then remove pixels that are not connected with the center + if isempty(merged_ROI) + break; + end + end + fprintf('Time cost in updating spatial & temporal components: %.2f seconds\n', toc); + + %% pick neurons from the residual (cell 4). + if miter==1 + neuron.options.seed_method = 'auto'; % methods for selecting seed pixels {'auto', 'manual'} + [center_new, Cn_res, pnr_res] = neuron.pickNeurons(Ysignal - neuron.A*neuron.C, patch_par); % method can be either 'auto' or 'manual' + end + + %% stop the iteration + temp = size(neuron.C, 1); + if or(nC==temp, miter==maxIter) + break; + else + miter = miter+1; + nC = temp; + end +end + +%% apply results to the full resolution +if or(ssub>1, tsub>1) + neuron_ds = neuron.copy(); % save the result + neuron = neuron_full.copy(); + cnmfe_full; + neuron_full = neuron.copy(); +end + + +%% delete some neurons and run CNMF-E iteration +neuron.viewNeurons([], neuron.C_raw); +tic; +cnmfe_update_BG; +fprintf('Time cost in updating the background: %.2f seconds\n', toc); +%update spatial & temporal components +tic; +for m=1:2 + %temporal + neuron.updateTemporal_endoscope(Ysignal); + cnmfe_quick_merge; % run neuron merges + %spatial + neuron.updateSpatial_endoscope(Ysignal, Nspatial, update_spatial_method); + neuron.trimSpatial(0.01, 3); % for each neuron, apply imopen first and then remove pixels that are not connected with the center +end +fprintf('Time cost in updating spatial & temporal components: %.2f seconds\n', toc); + +%% display neurons +dir_neurons = sprintf('%s%s%s_neurons%s', dir_nm, filesep, file_nm, filesep); +if exist('dir_neurons', 'dir') + temp = cd(); + cd(dir_neurons); + delete *; + cd(temp); +else + mkdir(dir_neurons); +end +neuron.viewNeurons([], neuron.C_raw, dir_neurons); +close(gcf); + +%% display contours of the neurons +neuron.Coor = neuron.get_contours(0.8); % energy within the contour is 80% of the total +figure; +Cnn = correlation_image(neuron.reshape(Ysignal(:, 1:5:end), 2), 4); +neuron.Coor = plot_contours(neuron.A, Cnn, 0.8, 1, [], neuron.Coor, 2); +colormap winter; +axis equal; axis off; +title('contours of estimated neurons'); + +% plot contours with IDs +% [Cn, pnr] = neuron.correlation_pnr(Y(:, round(linspace(1, T, min(T, 1000))))); +figure; +Cn = imresize(Cn, [d1, d2]); +plot_contours(neuron.A, Cn, 0.8, 1, [], neuron.Coor, 2); +colormap winter; +title('contours of estimated neurons'); + +%% check spatial and temporal components by playing movies +save_avi = false; +avi_name = 'play_movie.avi'; +neuron.Cn = Cn; +neuron.runMovie(Ysignal, [0, 50], save_avi, avi_name); + +%% save video +kt = 3; % play one frame in every kt frames +save_avi = true; +center_ac = median(max(neuron.A,[],1)'.*max(neuron.C,[],2)); % the denoised video are mapped to [0, 2*center_ac] of the colormap +cnmfe_save_video; + +%% save results +globalVars = who('global'); +eval(sprintf('save %s%s%s_results.mat %s', dir_nm, filesep, file_nm, strjoin(globalVars))); \ No newline at end of file diff --git a/ca_source_extraction/endoscope/extract_ac.m b/ca_source_extraction/endoscope/extract_ac.m index 5be628a..ada3bbe 100644 --- a/ca_source_extraction/endoscope/extract_ac.m +++ b/ca_source_extraction/endoscope/extract_ac.m @@ -66,12 +66,14 @@ end ai = ai(:); -% %% threshold the spatial shape and remove outliers -% % remove outliers -% temp = full(ai>quantile(ai(:), 0.5)); -% l = bwlabel(reshape(temp, nr, nc), 4); -% temp(l~=l(ind_ctr)) = false; -% ai(~temp(:)) = 0; + +%% threshold the spatial shape and remove outliers +% remove outliers +temp = full(ai>quantile(ai(:), 0.5)); +l = bwlabel(reshape(temp, nr, nc), 4); +temp(l~=l(ind_ctr)) = false; +ai(~temp(:)) = 0; + if sum(ai(:)>0) < min_pixels %the ROI is too small ind_success=false; return; diff --git a/ca_source_extraction/endoscope/greedyROI_endoscope.m b/ca_source_extraction/endoscope/greedyROI_endoscope.m index 90e718b..3074bac 100644 --- a/ca_source_extraction/endoscope/greedyROI_endoscope.m +++ b/ca_source_extraction/endoscope/greedyROI_endoscope.m @@ -128,6 +128,7 @@ % HY_med = median(HY, 2); % HY_max = max(HY, [], 2)-HY_med; % maximum projection HY = bsxfun(@minus, HY, median(HY, 2)); +HY0=HY; % saved as intact filtered result. HY_max = max(HY, [], 2); Ysig = GetSn(HY); PNR = reshape(HY_max./Ysig, d1, d2); @@ -451,7 +452,7 @@ end end center = center(1:k, :); -results.Ain = sparse(Ain(:, 1:k)); +results.Ain = Ain(:, 1:k); results.Cin = Cin(1:k, :); results.Cin_raw = Cin_raw(1:k, :); if deconv_flag diff --git a/ca_source_extraction/utilities/HALS_spatial.m b/ca_source_extraction/utilities/HALS_spatial.m index 8eb96e1..c2da99c 100644 --- a/ca_source_extraction/utilities/HALS_spatial.m +++ b/ca_source_extraction/utilities/HALS_spatial.m @@ -39,7 +39,7 @@ continue; end tmp_ind = active_pixel(:, k); - ak = max(0, A(tmp_ind, k)+(U(tmp_ind, k)-A(tmp_ind,:)*V(:, k))/cc(k)); + ak = max(0, full(A(tmp_ind, k))+(U(tmp_ind, k)-full(A(tmp_ind,:))*V(:, k))/cc(k)); A(tmp_ind, k) = ak; end end \ No newline at end of file diff --git a/ca_source_extraction/utilities/order_ROIs.m b/ca_source_extraction/utilities/order_ROIs.m index d83428a..5d4078e 100644 --- a/ca_source_extraction/utilities/order_ROIs.m +++ b/ca_source_extraction/utilities/order_ROIs.m @@ -1,4 +1,4 @@ -function [A_or,C_or,S_or,P_or,srt] = order_ROIs(A,C,S,P, srt) +function [A_or,C_or,S_or,P_or,srt] = order_ROIs(A,C,S,P,srt) % ordering of the found components based on their maximum temporal % activation and their size (through their l_inf norm) @@ -22,11 +22,13 @@ P_or = []; else P_or = P; + try if isfield(P,'gn')&& ~isempty(P.gn); P_or.gn=P.gn(srt); end if isfield(P,'b')&& ~isempty(P.b); P_or.b=P.b(srt); end if isfield(P,'c1')&&~isempty(P.c1); P_or.c1=P.c1(srt); end if isfield(P,'neuron_sn')&&~isempty(P.neuron_sn); P_or.neuron_sn=P.neuron_sn(srt); end + if isfield(P,'THRESH')&&~isempty(P.THRESH); P_or.THRESH.Corr=P.THRESH.Corr(srt); P_or.THRESH.PNR=P.THRESH.PNR(srt); end catch end end diff --git a/cnmfe_scripts/cnmfe_choose_data.m b/cnmfe_scripts/cnmfe_choose_data.m index abb1653..b6fec78 100644 --- a/cnmfe_scripts/cnmfe_choose_data.m +++ b/cnmfe_scripts/cnmfe_choose_data.m @@ -13,7 +13,7 @@ [dir_nm, file_nm, file_type] = fileparts(nam); else % use pre-specified file - if exist(nam, 'file') + if 1; %exist(nam, 'file') [dir_nm, file_nm, file_type] = fileparts(nam); else dir_nm = 0; @@ -59,7 +59,8 @@ %% information of the data data = matfile(nam_mat); -Ysiz = data.Ysiz; + +Ysiz = size(data.Y); %data.Ysiz; d1 = Ysiz(1); %height d2 = Ysiz(2); %width numFrame = Ysiz(3); %total number of frames diff --git a/cnmfe_scripts/cnmfe_load_data.m b/cnmfe_scripts/cnmfe_load_data.m index 8184b83..9dff147 100644 --- a/cnmfe_scripts/cnmfe_load_data.m +++ b/cnmfe_scripts/cnmfe_load_data.m @@ -1,6 +1,9 @@ if and(ssub==1, tsub==1) neuron = neuron_full; - Y = double(data.Y(:, :, sframe+(1:num2read)-1)); + + tmpY = data.Y; + + Y = single(tmpY(:, :, sframe+(1:num2read)-1)); [d1s,d2s, T] = size(Y); fprintf('\nThe data has been loaded into RAM. It has %d X %d pixels X %d frames. \nLoading all data requires %.2f GB RAM\n\n', d1s, d2s, T, d1s*d2s*T*8/(2^30)); else diff --git a/cnmfe_scripts/cnmfe_quick_merge.m b/cnmfe_scripts/cnmfe_quick_merge.m index 5830198..fad9bc0 100644 --- a/cnmfe_scripts/cnmfe_quick_merge.m +++ b/cnmfe_scripts/cnmfe_quick_merge.m @@ -46,9 +46,18 @@ neuron.C_raw = [neuron.C_raw(~ind_after, :); neuron_bk.C_raw(ind_before, :)]; neuron.S = [neuron.S(~ind_after, :); neuron_bk.S(ind_before, :)]; neuron.P.kernel_pars = [neuron.P.kernel_pars(~ind_after, :); neuron_bk.P.kernel_pars(ind_before, :)]; - close; + + neuron.P.THRESH.Corr=[neuron.P.THRESH.Corr(~ind_after),neuron.P.THRESH.Corr(ind_before)]; + neuron.P.THRESH.PNR=[neuron.P.THRESH.PNR(~ind_after),neuron.P.THRESH.PNR(ind_before)]; + close; + end +% sort neurons +[Cpnr, srt] = sort(max(neuron.C, [], 2).*max(neuron.A, [], 1)', 'descend'); +neuron.orderROIs(srt); + + %% view neurons if view_neurons neuron.viewNeurons([], neuron.C_raw); diff --git a/cnmfe_scripts/cnmfe_update_BG.m b/cnmfe_scripts/cnmfe_update_BG.m index d63a6be..ce2690a 100644 --- a/cnmfe_scripts/cnmfe_update_BG.m +++ b/cnmfe_scripts/cnmfe_update_BG.m @@ -1,5 +1,5 @@ clear Ysignal; -tic; +tic; Ybg = Y-neuron.A*neuron.C; rr = ceil(neuron.options.gSiz * bg_neuron_ratio); active_px = []; %(sum(IND, 2)>0); %If some missing neurons are not covered by active_px, use [] to replace IND diff --git a/deconvolveCa/deconvolveCa.m b/deconvolveCa/deconvolveCa.m index 75ed71d..c8d9130 100644 --- a/deconvolveCa/deconvolveCa.m +++ b/deconvolveCa/deconvolveCa.m @@ -78,8 +78,8 @@ catch c = y*0; s = c; - fprintf('fail to deconvolve the trace\n'); - return; + fprintf('fail to deconvolve the trace\n'); + return end if length(options.pars)~=1 c = zeros(size(y)); diff --git a/deconvolveCa/deconvolveCa_NoNoise.m b/deconvolveCa/deconvolveCa_NoNoise.m new file mode 100644 index 0000000..a580264 --- /dev/null +++ b/deconvolveCa/deconvolveCa_NoNoise.m @@ -0,0 +1,332 @@ +function [c, s, options,ind_success] = deconvolveCa_NoNoise(y, varargin) +%% infer the most likely discretized spike train underlying an fluorescence trace +%% Solves mutliple formulation of the problem +% 1) FOOPSI, +% mininize_{c,s} 1/2 * norm(y-c,2)^2 + lambda * norm(s,1) +% subject to c>=0, s>=0, s=Gc +% 2) constrained FOOPSI +% minimize_{c,s} norm(s, q) +% subject to norm(y-c,2) <= sn*sqrt(T), c>=0, s>=0, s=Gc +% where q is either 1 or 0, rendering the problem convex or non-convex. +% 3) hard threshrinkage +% minimize_{c,s} 1/2 * norm(y-c, 2)^2 +% subjec to c>=0, s=Gc, s=0 or s>=smin +% 4) Nonnegative least square problem (NNLS) +% min_{s} norm(y - s*h, 2)^2 + lambda * norm(s,1) +% subject to s>=0 + +%% inputs: +% y: T x 1 vector, fluorescence trace +% varargin: variable input arguments +% type: string, defines the model of the deconvolution kernel. possible +% options are: +% 'ar1': auto-regressive model with order p=1 +% 'ar2': auto-regressive model with order p=2 +% 'exp2': the convolution kernel is modeled as the difference of two +% exponential functions - +% h(t) = (exp(-t/tau_d) - exp(-t/tau_r)) / (tau_d-tau_r) +% 'kernel': a vector of the convolution kernel +% pars: parameters for the specified convolution kernel. it has +% different shapes for differrent types of the convolution model: +% 'ar1': scalar +% 'ar2': 2 x 1 vector, [r_1, r_2] +% 'exp2': 2 x 1 vector, [tau_r, tau_d] +% 'kernel': maxISI x 1 vector, the kernel. +% sn: scalar, standard deviation of the noise distribution. If no +% values is give, then sn is estimated from the data based on power +% spectual density method. +% b: fluorescence baseline vlaues. default is 0 +% optimize_pars: estimate the parameters of the convolution kernel. default: 0 +% optimize_b: estimate the baseline. default: 0 +% lambda: penalty parameter +% method: methods for running deconvolution. {'foopsi', +% 'constrained_foopsi' (default), 'thresholded'}, + +%% outputs: +% c: T x 1 vector, denoised trace +% s: T x 1 vector, deconvolved signal +% b: fluorescence baseline +% kernel: struct variable containing the parameters for the selected +% convolution model +% lambda: Optimal Lagrange multiplier for noise constraint under L1 penalty +% """olves the noise constrained sparse nonnegat + +%% Authors: Pengcheng Zhou, Carnegie Mellon University, 2016 +% ported from the Python implementation from Johannes Friedrich + +%% References +% Friedrich J et.al., NIPS 2016, Fast Active Set Method for Online Spike Inference from Calcium Imaging + +%% input arguments +y = reshape(y, [], 1); % reshape the trace as a vector +options = parseinputs(varargin{:}); % parse input arguments +if isempty(y) + c = []; s = []; + return; +end +win = options.window; % length of the convolution kernel + +% estimate time constant +ind_success=false; +k=1; +if isempty(options.pars) + while and(k<2,~ind_success) + switch options.type + case 'ar1' + options.pars = estimate_time_constant_NoNoise(y, 1); + if length(options.pars)~=1 + display('ar1 is not suitable, trying ar2.') + options.type='ar2'; + k=k+1; + else + ind_success=true; + end + case 'ar2' + options.pars = estimate_time_constant_NoNoise(y, 2); + if length(options.pars)~=2 + display('ar2 is not suitable, trying ar1.') + options.type='ar1'; + k=k+1; + else + ind_success=true; + end + case 'exp2' + g = estimate_time_constant_NoNoise(y, 2); + if length(options.pars)~=2 + display('exp2 is not suitable, trying ar1.') + options.type='ar1'; + k=k+1; + else + options.pars = ar2exp(g); + ind_success=true; + end + case 'kernel' + g = estimate_time_constant_NoNoise(y, 2); + if length(options.pars)~=2 + display('exp2 is not suitable, trying other methods.') + options.type='ar1'; + k=k+1; + else + taus = ar2exp(g); + options.pars = exp2kernel(taus, options.win); % convolution kernel + ind_success=true; + end + end + end +end + +if ind_success==false + c = ci_raw; + s = []; + options.pars = []; +else % run deconvolution + c = y; + s = y; + switch lower(options.method) + case 'foopsi' %% use FOOPSI + if strcmpi(options.type, 'ar1') % AR 1 + [c, s, options.b, options.pars] = foopsi_oasisAR1(y-options.b, options.pars, options.lambda, ... + options.smin, options.optimize_b, options.optimize_pars, [], options.maxIter); + elseif strcmpi(options.type, 'ar2') % AR 2 + [c, s, options.b, options.pars] = foopsi_oasisAR2(y-options.b, options.pars, options.lambda, ... + options.smin); + elseif strcmpi(options.type, 'exp2') % difference of two exponential functions + kernel = exp2kernel(options.pars, options.window); + [c, s] = onnls(y-options.b, kernel, options.lambda, ... + options.shift, options.window); + elseif strcmpi(options.type, 'kernel') % convolution kernel itself + [c, s] = onnls(y-options.b, options.pars, options.lambda, ... + options.shift, options.window); + else + disp('to be done'); + end + case 'constrained' + if strcmpi(options.type, 'ar1') % AR1 + [c, s, options.b, options.pars, options.lambda] = constrained_oasisAR1(y,... + options.pars, options.sn, options.optimize_b, options.optimize_pars, ... + [], options.maxIter); + else + [cc, options.b, c1, options.pars, options.sn, s] = constrained_foopsi(y,[],[],options.pars,options.sn, ... + options.extra_params); + gd = max(roots([1,-options.pars'])); % decay time constant for initial concentration + gd_vec = gd.^((0:length(y)-1)); + c = cc(:) + c1*gd_vec'; + options.cin = c1; + end + case 'thresholded' %% Use hard-shrinkage method + if strcmpi(options.type, 'ar1') + [c, s, options.b, options.pars, options.smin] = thresholded_oasisAR1(y,... + options.pars, options.sn, options.optimize_b, options.optimize_pars, ... + [], options.maxIter, options.thresh_factor, options.p_noise); + % if and(options.smin==0, options.optimize_smin) % smin is given + % [c, s, options.b, options.pars, options.smin] = thresholded_oasisAR1(y,... + % options.pars, options.sn, options.optimize_b, options.optimize_pars, ... + % [], options.maxIter, options.thresh_factor); + % else + % [c, s] = oasisAR1(y-options.b, options.pars, options.lambda, ... + % options.smin); + % end + elseif strcmpi(options.type, 'ar2') + [c, s, options.b, options.pars, options.smin] = thresholded_oasisAR2(y,... + options.pars, options.sn, options.smin, options.optimize_b, options.optimize_pars, ... + [], options.maxIter, options.thresh_factor); + % if and(options.smin==0, options.optimize_smin) % smin is given + % [c, s, options.b, options.pars, options.smin] = thresholded_oasisAR2(y,... + % options.pars, options.sn, options.optimize_b, options.optimize_pars, ... + % [], options.maxIter, options.thresh_factor); + % else + % [c, s] = oasisAR2(y-options.b, options.pars, options.lambda, ... + % options.smin); + % end + elseif strcmpi(options.type, 'exp2') % difference of two exponential functions + d = options.pars(1); + r = options.pars(2); + options.pars = (exp(log(d)*(1:win)) - exp(log(r)*(1:win))) / (d-r); % convolution kernel + [c, s] = onnls(y-options.b, options.pars, options.lambda, ... + options.shift, options.window, [], [], [], options.smin); + elseif strcmpi(options.type, 'kernel') % convolution kernel itself + [c, s] = onnls(y-options.b, options.pars, options.lambda, ... + options.shift, options.window, [], [], [], options.smin); + else + disp('to be done'); + end + case 'mcmc' + SAMP = cont_ca_sampler(y,options.extra_params); + options.extra_params = SAMP; + options.mcmc_results = SAMP; + plot_continuous_samples(SAMP,y); + end +end + +function options=parseinputs(varargin) +%% parse input variables + +%% default options +options.type = 'ar1'; +options.pars = []; +options.sn = []; +options.b = 0; +options.lambda = 0; +options.optimize_b = false; +options.optimize_pars = false; +options.optimize_smin = false; +options.method = 'constrained'; +options.window = 200; +options.shift = 100; +options.smin = 0; +options.maxIter = 10; +options.thresh_factor = 1.0; +options.extra_params = []; +options.p_noise = 0.9999; + +if isempty(varargin) + return; +elseif isstruct(varargin{1}) && ~isempty(varargin{1}) + tmp_options = varargin{1}; + field_nams = fieldnames(tmp_options); + for m=1:length(field_nams) + eval(sprintf('options.%s=tmp_options.%s;', field_nams{m}, field_nams{m})); + end + k = 2; +else + k = 1; +end +%% parse all input arguments +while k<=nargin + + switch lower(varargin{k}) + case {'ar1', 'ar2', 'exp2', 'kernel'} + % convolution kernel type + options.type = lower(varargin{k}); + if (k1) && p < 5 +% warning('No stable AR(%i) model found. Checking for AR(%i) model \n',p, p+1); + p = p + 1; + g = estimate_time_constant_NoNoise(y,p,lags); +end +if p == 5 + g = 0; +end + +% re-adjust time constant values +rg = roots([1;-g(:)]); +if ~isreal(rg); rg = real(rg) + .001*randn(size(rg)); end +rg(rg>1) = 0.95 + 0.001*randn(size(rg(rg>1))); +rg(rg<0) = 0.15 + 0.001*randn(size(rg(rg<0))); +pg = poly(fudge_factor*rg); +g = -pg(2:end); + diff --git a/demos/demo_batch_1p.m b/demos/demo_batch_1p.m index f724b37..95eba50 100644 --- a/demos/demo_batch_1p.m +++ b/demos/demo_batch_1p.m @@ -3,20 +3,27 @@ %% choose multiple datasets or just one neuron = Sources2D(); -nams = {'./data_1p.tif'}; % you can put all file names into a cell array; when it's empty, manually select files +%nams = {'./data_1p.tif'}; % you can put all file names into a cell array; when it's empty, manually select files +nams={}; + +datadir='Z:\EmilyShijieShared_old\PC_batch\'; +for i=1:numel(samplelist) + nams{i}=[datadir,samplelist(i).name]; +end + nams = neuron.select_multiple_files(nams); %if nam is [], then select data interactively %% parameters % ------------------------- COMPUTATION ------------------------- % pars_envs = struct('memory_size_to_use', 8, ... % GB, memory space you allow to use in MATLAB 'memory_size_per_patch', 0.5, ... % GB, space for loading data within one patch - 'patch_dims', [64, 64],... %GB, patch size + 'patch_dims', [300, 400],... %GB, patch size 'batch_frames', 1000); % number of frames per batch % ------------------------- SPATIAL ------------------------- % -gSig = 3; % pixel, gaussian width of a gaussian kernel for filtering the data. 0 means no filtering -gSiz = 13; % pixel, neuron diameter +gSig = 10; % pixel, gaussian width of a gaussian kernel for filtering the data. 0 means no filtering +gSiz = 17; % pixel, neuron diameter ssub = 1; % spatial downsampling factor -with_dendrites = true; % with dendrites or not +with_dendrites = false; % with dendrites or not if with_dendrites % determine the search locations by dilating the current neuron shapes updateA_search_method = 'dilate'; %#ok @@ -32,16 +39,16 @@ spatial_algorithm = 'hals'; % ------------------------- TEMPORAL ------------------------- % -Fs = 10; % frame rate +Fs = 30; % frame rate tsub = 1; % temporal downsampling factor deconv_options = struct('type', 'ar1', ... % model of the calcium traces. {'ar1', 'ar2'} - 'method', 'foopsi', ... % method for running deconvolution {'foopsi', 'constrained', 'thresholded'} + 'method', 'constrained', ... % method for running deconvolution {'foopsi', 'constrained', 'thresholded'} 'smin', -5, ... % minimum spike size. When the value is negative, the actual threshold is abs(smin)*noise level 'optimize_pars', true, ... % optimize AR coefficients 'optimize_b', true, ...% optimize the baseline); 'max_tau', 100); % maximum decay time (unit: frame); -nk = 3; % detrending the slow fluctuation. usually 1 is fine (no detrending) +nk = 1; % detrending the slow fluctuation. usually 1 is fine (no detrending) % when changed, try some integers smaller than total_frame/(Fs*30) detrend_method = 'spline'; % compute the local minimum as an estimation of trend. @@ -64,7 +71,7 @@ % ------------------------- INITIALIZATION ------------------------- % K = []; % maximum number of neurons per patch. when K=[], take as many as possible. min_corr = 0.8; % minimum local correlation for a seeding pixel -min_pnr = 8; % minimum peak-to-noise ratio for a seeding pixel +min_pnr = 21; % minimum peak-to-noise ratio for a seeding pixel min_pixel = gSig^2; % minimum number of nonzero pixels for each neuron bd = 0; % number of rows/columns to be ignored in the boundary (mainly for motion corrected data) frame_range = []; % when [], uses all frames diff --git a/demos/demo_endoscope_SingANDSleep.m b/demos/demo_endoscope_SingANDSleep.m new file mode 100644 index 0000000..50e6cdf --- /dev/null +++ b/demos/demo_endoscope_SingANDSleep.m @@ -0,0 +1,260 @@ +%% Compile singing/sleep data in one day +Datafolder='/net/feevault/data0/elm/ProcessedCalciumData/sleep/6962/042917/'; +singing = matfile(fullfile(Datafolder,'compiled.mat')); +sleep=matfile(fullfile(Datafolder,'CompiledSleep_6962_20170429_105506-20170430_104328.mat')); +Myfolder='/home/shijiegu/SingAndSleep'; + + +Bird=sleep.Bird; +SleepStart=sleep.SleepStart; +Day=datestr(SleepStart(1:13),'ddmmmyyyy'); + +Y = cat(3,singing.Y,sleep.Y); +display('Y cated') +VIDEOfs=singing.VIDEOfs; +FILE = fullfile(Myfolder, [Bird '_' 'SingANDSleep_' Day]); +save(FILE, 'VIDEOfs','Y','-v7.3'); + +%% Demo Starts +% parameter sweep version +global d1 d2 numFrame ssub tsub sframe num2read Fs neuron neuron_ds ... + neuron_full Ybg_weights outputfolder data Ysiz nam min_pnr gSiz gSig; %#ok % global variables, don't change them manually + +addpath(genpath('/home/shijiegu/CNMF_E-master')) + +%% select data and map it to the RAM +%nam = '/Users/gushijie/Documents/MATLAB/CalciumExtraction/CaELM_row3888.mat'; +nam=FILE; +%nam='/Volumes/data0-shared/elm/ProcessedCalciumData/6865_Jan9/compiled.mat'; +%cnmfe_choose_data; +[dir_nm, file_nm, file_type] = fileparts(nam); +nam_mat = nam; + +% information of the data +data = matfile(nam_mat); +Ysiz = size(data.Y); +d1 = Ysiz(1); %height +d2 = Ysiz(2); %width +numFrame = Ysiz(3); %total number of frames + +fprintf('\nThe data has been mapped to RAM. It has %d X %d pixels X %d frames. \nLoading all data requires %.2f GB RAM\n\n', d1, d2, numFrame, prod(Ysiz)*8/(2^30)); +%% create Source2D class object for storing results and parameters +Fs = 30; % frame rate +ssub = 1; % spatial downsampling factor +tsub = 1; % temporal downsampling factor +gSig = gSig; % width of the gaussian kernel, which can approximates the average neuron shape +gSiz = gSiz; % maximum diameter of neurons in the image plane. larger values are preferred. +neuron_full = Sources2D('d1',d1,'d2',d2, ... % dimensions of datasets + 'ssub', ssub, 'tsub', tsub, ... % downsampleing + 'gSig', gSig,... % sigma of the 2D gaussian that approximates cell bodies + 'gSiz', gSiz); % average neuron size (diameter) +neuron_full.Fs = Fs; % frame rate + +% with dendrites or not +with_dendrites = false; +if with_dendrites + % determine the search locations by dilating the current neuron shapes + neuron_full.options.search_method = 'dilate'; + neuron_full.options.bSiz = 20; +else + % determine the search locations by selecting a round area + neuron_full.options.search_method = 'ellipse'; + neuron_full.options.dist = 5; +end + +%% options for running deconvolution +neuron_full.options.deconv_options = struct('type', 'ar1', ... % model of the calcium traces. {'ar1', 'ar2'} + 'method', 'thresholded', ... % method for running deconvolution {'foopsi', 'constrained', 'thresholded'} + 'optimize_pars', true, ... % optimize AR coefficients + 'optimize_b', false, ... % optimize the baseline + 'optimize_smin', true); % optimize the threshold + +%% downsample data for fast and better initialization +sframe=1; % user input: first frame to read (optional, default:1) +num2read= numFrame; % user input: how many frames to read (optional, default: until the end) + +tic; +cnmfe_load_data; +fprintf('Time cost in downsapling data: %.2f seconds\n', toc); + +Y = neuron.reshape(Y, 1); % convert a 3D video into a 2D matrix + +%% compute correlation image and peak-to-noise ratio image. +cnmfe_show_corr_pnr; % this step is not necessary, but it can give you some... + % hints on parameter selection, e.g., min_corr & min_pnr + +%% initialization of A, C +% parameters +debug_on = false; % visualize the initialization procedue. +save_avi = false; %save the initialization procedure as an avi movie. +patch_par = [1,1]*1; %1; % divide the optical field into m X n patches and do initialization patch by patch. It can be used when the data is too large +K = []; % maximum number of neurons to search within each patch. you can use [] to search the number automatically + +min_corr = 0.85; % minimum local correlation for a seeding pixel +min_pnr = min_pnr; % minimum peak-to-noise ratio for a seeding pixel +min_pixel = 5; % minimum number of nonzero pixels for each neuron +bd = 1; % number of rows/columns to be ignored in the boundary (mainly for motion corrected data) +neuron.updateParams('min_corr', min_corr, 'min_pnr', min_pnr, ... + 'min_pixel', min_pixel, 'bd', bd); +neuron.options.nk = 1; % number of knots for detrending + +% greedy method for initialization +tic; +[center, Cn, pnr] = neuron.initComponents_endoscope(Y, K, patch_par, debug_on, save_avi); +fprintf('Time cost in initializing neurons: %.2f seconds\n', toc); + +% show results +% figure; +% imagesc(Cn, [0.1, 0.95]); +% hold on; plot(center(:, 2), center(:, 1), 'or'); +% colormap; axis off tight equal; + +% sort neurons +[~, srt] = sort(max(neuron.C, [], 2), 'descend'); +neuron.orderROIs(srt); +neuron_init = neuron.copy(); +display('line 99') +%% iteratively update A, C and B +% parameters, merge neurons +display_merge = false; % visually check the merged neurons +view_neurons = false; % view all neurons + +% parameters, estimate the background +spatial_ds_factor = 1; % spatial downsampling factor. it's for faster estimation +thresh = 10; % threshold for detecting frames with large cellular activity. (mean of neighbors' activity + thresh*sn) + +bg_neuron_ratio = 1; % spatial range / diameter of neurons + +% parameters, estimate the spatial components +update_spatial_method = 'hals'; % the method for updating spatial components {'hals', 'hals_thresh', 'nnls', 'lars'} +Nspatial = 5; % this variable has different meanings: + %1) udpate_spatial_method=='hals' or 'hals_thresh', + %then Nspatial is the maximum iteration + %2) update_spatial_method== 'nnls', it is the maximum + %number of neurons overlapping at one pixel + +% parameters for running iteratiosn +nC = size(neuron.C, 1); % number of neurons + +maxIter = 2; % maximum number of iterations +miter = 1; +display('line 114') +while miter <= maxIter + %% merge neurons, order neurons and delete some low quality neurons + if miter ==1 + merge_thr = [1e-1, 0.8, .1]; % thresholds for merging neurons + % corresponding to {sptial overlaps, temporal correlation of C, + %temporal correlation of S} + else + merge_thr = [0.6, 0.5, 0.1]; + end + % merge neurons + cnmfe_quick_merge; % run neuron merges + + %% udpate background (cell 1, the following three blocks can be run iteratively) + % estimate the background + tic; + cnmfe_update_BG; + fprintf('Time cost in estimating the background: %.2f seconds\n', toc); + % neuron.playMovie(Ysignal); % play the video data after subtracting the background components. + + %% update spatial & temporal components + tic; + for m=1:2 + %temporal + neuron.updateTemporal_endoscope(Ysignal); + cnmfe_quick_merge; % run neuron merges + %spatial + neuron.updateSpatial_endoscope(Ysignal, Nspatial, update_spatial_method); + neuron.trimSpatial(0.01, 3); % for each neuron, apply imopen first and then remove pixels that are not connected with the center + if isempty(merged_ROI) + break; + end + end + fprintf('Time cost in updating spatial & temporal components: %.2f seconds\n', toc); + + %% pick neurons from the residual (cell 4). + if miter==1 + neuron.options.seed_method = 'auto'; % methods for selecting seed pixels {'auto', 'manual'} + [center_new, Cn_res, pnr_res] = neuron.pickNeurons(Ysignal - neuron.A*neuron.C, patch_par); % method can be either 'auto' or 'manual' + end + + %% stop the iteration + temp = size(neuron.C, 1); + if or(nC==temp, miter==maxIter) + break; + else + miter = miter+1; + nC = temp; + end +end + +%% apply results to the full resolution +if or(ssub>1, tsub>1) + neuron_ds = neuron.copy(); % save the result + neuron = neuron_full.copy(); + cnmfe_full; + neuron_full = neuron.copy(); +end + + +%% delete some neurons and run CNMF-E iteration +% neuron.viewNeurons([], neuron.C_raw); +tic; +cnmfe_update_BG; +fprintf('Time cost in estimating the background: %.2f seconds\n', toc); +%update spatial & temporal components +tic; +for m=1:2 + %temporal + neuron.updateTemporal_endoscope(Ysignal); + cnmfe_quick_merge; % run neuron merges + %spatial + neuron.updateSpatial_endoscope(Ysignal, Nspatial, update_spatial_method); + neuron.trimSpatial(0.01, 3); % for each neuron, apply imopen first and then remove pixels that are not connected with the center +end +fprintf('Time cost in updating spatial & temporal components: %.2f seconds\n', toc); + +%% display neurons + +teststring=sprintf('%s_%s_singsleepresults.mat', Bird, Day); +neuron.viewNeurons([], neuron.C_raw, teststring); +close(gcf); + +ColorAllNeurons(neuron.A) +fprintf('Almost done'); + +% %% display contours of the neurons +% neuron.Coor = neuron.get_contours(0.8); % energy within the contour is 80% of the total +% figure; +% Cnn = correlation_image(neuron.reshape(Ysignal(:, 1:5:end), 2), 4); +% neuron.Coor = plot_contours(neuron.A, Cnn, 0.8, 0, [], neuron.Coor, 2); +% colormap winter; +% axis equal; axis off; +% title('contours of estimated neurons'); +% +% plot contours with IDs +% [Cn, pnr] = neuron.correlation_pnr(Y(:, round(linspace(1, T, min(T, 1000))))); +% figure; +% Cn = imresize(Cn, [d1, d2]); +% plot_contours(neuron.A, Cn, 0.8, 0, [], neuron.Coor, 2); +% colormap winter; +% title('contours of estimated neurons'); + +%% check spatial and temporal components by playing movies +% save_avi = false; +% avi_name = 'play_movie.avi'; +% neuron.Cn = Cn; +% neuron.runMovie(Ysignal, [0, 50], save_avi, avi_name); +% +% %% save video +% kt = 3; % play one frame in every kt frames +% save_avi = true; +% center_ac = median(max(neuron.A,[],1)'.*max(neuron.C,[],2)); % the denoised video are mapped to [0, 2*center_ac] of the colormap +% cnmfe_save_video; + +%% save results +globalVars = who('global'); +eval(sprintf('save teststring %s',strjoin(globalVars))); +fprintf('Whole thing saved and done.'); + diff --git a/testing.m b/testing.m new file mode 100644 index 0000000..22bded2 --- /dev/null +++ b/testing.m @@ -0,0 +1,3 @@ +function a=testing(x,y) +a=x+y; +end \ No newline at end of file