Merge pull request #16 from akeshavan/master

fix: QA and correlation matrix saved

Author: satra

fmri/qa/QA_fmri.py

@@ -14,7 +14,7 @@
 from nipype.interfaces import fsl
 #from nipype.utils.config import config
 #config.enable_debug_mode()
-from QA_utils import plot_ADnorm, tsdiffana, tsnr_roi, combine_table, art_output, plot_motion
+from QA_utils import plot_ADnorm, tsdiffana, tsnr_roi, combine_table, art_output, plot_motion, plot_ribbon, plot_anat
 import sys
 sys.path.insert(0,'../../utils/')
 from reportsink.io import ReportSink
@@ -45,6 +45,7 @@ def preproc_datagrabber(name='preproc_datagrabber'):
                          name = name)
     datasource.inputs.base_directory = os.path.join(c.sink_dir,'analyses','func')
     datasource.inputs.template ='*'
+    datasource.sort_filelist = True
     datasource.inputs.field_template = dict(motion_parameters='%s/preproc/motion/*.par',
                                             outlier_files='%s/preproc/art/*_outliers.txt',
                                             art_norm='%s/preproc/art/norm.*.txt',
@@ -268,11 +269,24 @@ def QA_workflow(name='QA'):
                         name='tsdiffana', iterfield=["img"])
 
     art_info = pe.MapNode(util.Function(input_names = ['art_file'], 
-                                      output_names = ['table'], 
+                                      output_names = ['table','out'], 
                                       function=art_output), 
                         name='art_output', iterfield=["art_file"])
 
     fssource = pe.Node(interface = FreeSurferSource(),name='fssource')
+    
+    plotribbon = pe.Node(util.Function(input_names=['Brain'],
+                                      output_names=['images'],
+                                      function=plot_ribbon),
+                        name="plot_ribbon")
+    
+    workflow.connect(fssource, 'ribbon', plotribbon, 'Brain')
+    
+    
+    plotanat = pe.Node(util.Function(input_names=['brain'],
+                                      output_names=['images'],
+                                      function=plot_anat),
+                        name="plot_anat")
 
     roidevplot = tsnr_roi(plot=False,name='tsnr_stddev_roi',roi=['all'])
     roidevplot.inputs.inputspec.TR = c.TR
@@ -285,6 +299,7 @@ def QA_workflow(name='QA'):
     workflow.connect(infosource, 'subject_id', roidevplot, 'inputspec.subject')
     workflow.connect(infosource, 'subject_id', roisnrplot, 'inputspec.subject')
 
+   
     tablecombine = pe.MapNode(util.Function(input_names = ['roidev',
                                                         'roisnr'],
                                          output_names = ['roisnr'], 
@@ -293,12 +308,12 @@ def QA_workflow(name='QA'):
 
 
 
-    adnormplot = pe.MapNode(util.Function(input_names = ['ADnorm','TR'], 
+    adnormplot = pe.MapNode(util.Function(input_names = ['ADnorm','TR','norm_thresh','out'], 
                                        output_names = ['plot'], 
                                        function=plot_ADnorm), 
-                         name='ADnormplot', iterfield=['ADnorm'])
-    
-    
+                         name='ADnormplot', iterfield=['ADnorm','out'])
+    adnormplot.inputs.norm_thresh = c.norm_thresh
+    workflow.connect(art_info,'out',adnormplot,'out')
 
     convert = pe.Node(interface=fs.MRIConvert(),name='converter')
 
@@ -315,24 +330,17 @@ def QA_workflow(name='QA'):
                                           name='overlay_mask')
     overlaymask.inputs.threshold = 0
 
-    #overlaymask = pe.Node(interface=fsl.Overlay(),name='fsl_overlaymask')
-    #overlaymask.inputs.transparency = True
-    #overlaymask.inputs.stat_thresh = (0,1)
-    #overlaymask.inputs.full_bg_range= True
-    
-    #slicermask = pe.Node(interface=fsl.Slicer(),name='slicer_mask')
-    #slicermask.inputs.all_axial = True
-    #slicermask.inputs.image_width = 4
 
-    workflow.connect(datagrabber, 'mask', overlaymask, 'background_image')
-    workflow.connect(datagrabber, 'mean_image', overlaymask, 'stat_image')
+    #workflow.connect(datagrabber, 'mask', overlaymask, 'background_image')
+    workflow.connect(datagrabber, 'mean_image', plotanat, 'brain')
     #workflow.connect(overlaymask,'out_file', slicermask, 'in_file')
 
     write_rep = pe.Node(interface=ReportSink(orderfields=['Introduction',
                                                           'in_file',
                                                           'config_params',
                                                           'Art_Detect',
-                                                          'Brain_Mask_and_Mean_Functional',
+                                                          'Mean_Functional',
+                                                          'Ribbon',
                                                           'motion_plots',
                                                           'tsdiffana',
                                                           'ADnorm',
@@ -344,7 +352,7 @@ def QA_workflow(name='QA'):
     write_rep.inputs.report_name = "Preprocessing_Report"
     write_rep.inputs.json_sink = c.json_sink
     workflow.connect(infosource,'subject_id',write_rep,'container')
-    workflow.connect(overlaymask, 'fnames', write_rep, "Brain_Mask_and_Mean_Functional")
+    workflow.connect(plotanat, 'images', write_rep, "Mean_Functional")
     #workflow.connect(slicermask,'out_file', write_rep, "Brain_Mask_and_Mean_Functional")
 
     # Define Inputs
@@ -376,7 +384,7 @@ def QA_workflow(name='QA'):
     workflow.connect(convert,'out_file',voltransform,'target_file') 
     workflow.connect(inputspec,'reg_file',voltransform,'reg_file')
     workflow.connect(inputspec,'tsnr',voltransform, 'source_file')
-    
+    workflow.connect(plotribbon, 'images', write_rep, 'Ribbon')
     workflow.connect(voltransform,'transformed_file', overlaynew,'stat_image')
     workflow.connect(convert,'out_file', overlaynew,'background_image')
 
@@ -403,7 +411,7 @@ def QA_workflow(name='QA'):
     a.base_dir = c.working_dir
     a.write_graph()
     a.inputs.inputspec.config_params = start_config_table()
-    
+    a.config = {'execution' : {'crashdump_dir' : c.crash_dir}}
     if c.run_on_grid:
         a.run(plugin=c.plugin,plugin_args=c.plugin_args)
     else:

fmri/qa/QA_utils.py

@@ -18,10 +18,10 @@ def art_output(art_file):
     except:
         out=np.asarray([])
     table=[["file",art_file],["num outliers", str(out.shape)],["timepoints",str(out)]]
-    return table
+    return table, [out]
 
 
-def plot_ADnorm(ADnorm,TR):
+def plot_ADnorm(ADnorm,TR,norm_thresh,out):
     """ Returns a plot of the composite_norm file output from art
     
     Parameters
@@ -41,7 +41,7 @@ def plot_ADnorm(ADnorm,TR):
     import matplotlib.pyplot as plt
     import os
     import numpy as np
-    
+    out = out[0]
     plot = os.path.abspath('plot_'+os.path.split(ADnorm)[1]+'.png')
 
     data = np.genfromtxt(ADnorm)
@@ -50,7 +50,19 @@ def plot_ADnorm(ADnorm,TR):
     plt.plot(X,data)
     plt.xlabel('Time (s)')
     plt.ylabel('Composite Norm')
-    plt.savefig(plot)
+    
+    if norm_thresh > max(data):
+        plt.axis([0,TR*data.shape[0],0,norm_thresh*1.1])
+        plt.plot(X,np.ones(X.shape)*norm_thresh)
+        for o in out:
+            plt.plot(o*TR*np.ones(2),[0,norm_thresh*1.1],'r-')
+    else:
+        plt.axis([0,TR*data.shape[0],0,max(data)*1.1])
+        plt.plot(X,np.ones(X.shape)*norm_thresh)
+        for o in out:
+            plt.plot(o*TR*np.ones(2),[0,max(data)*1.1],'r-')
+    
+    plt.savefig(plot,bbox_inches='tight')
     plt.close()
     return plot
 
@@ -275,6 +287,7 @@ def plot_motion(motion_parameters):
     fname_t=os.path.abspath('translations.png')
     plt.figure(1,figsize = (8,3))
     plt.plot(np.genfromtxt(motion_parameters)[:,3:])
+    plt.legend(['x','y','z'])
     plt.title("Estimated Translations (mm)")
     plt.savefig(fname_t)
     plt.close()
@@ -283,8 +296,67 @@ def plot_motion(motion_parameters):
     plt.figure(2,figsize = (8,3))
     plt.plot(np.genfromtxt(motion_parameters)[:,:3])
     plt.title("Estimated Rotations (rad)")
+    plt.legend(['roll','pitch','yaw'])
     plt.savefig(fname_r)
     plt.close()
     fname = [fname_t, fname_r]
     return fname
 
+def plot_ribbon(Brain):
+    import os.path
+    import pylab as pl
+    from nibabel import load
+    from nipy.labs import viz   
+    images = []
+    
+    for brain in Brain:
+        if os.path.split(brain)[1] == 'ribbon.mgz':
+            img = load(brain)
+            data = img.get_data()*100
+            affine = img.get_affine() 
+            viz.plot_anat(anat=data, anat_affine=affine, draw_cross=False, slicer='x', cmap=viz.cm.black_green)
+            
+            x_view = os.path.abspath('x_view.png')
+            y_view = os.path.abspath('y_view.png')
+            z_view = os.path.abspath('z_view.png')
+            
+            pl.savefig(x_view,bbox_inches='tight')
+            
+            viz.plot_anat(anat=data, anat_affine=affine, draw_cross=False, slicer='y', cmap=viz.cm.black_green)
+            pl.savefig(y_view,bbox_inches='tight')
+            
+            viz.plot_anat(anat=data, anat_affine=affine, draw_cross=False, slicer='z', cmap=viz.cm.black_green)
+            pl.savefig(z_view,bbox_inches='tight')
+            
+            images = [x_view, y_view, z_view]
+            pl.close()
+    return images
+    
+def plot_anat(brain):
+    import os.path
+    import pylab as pl
+    from nibabel import load
+    from nipy.labs import viz   
+    
+    img = load(brain)
+    data = img.get_data()
+    affine = img.get_affine() 
+    viz.plot_anat(anat=data, anat_affine=affine, draw_cross=False, slicer='x')
+    
+    x_view = os.path.abspath('x_view.png')
+    y_view = os.path.abspath('y_view.png')
+    z_view = os.path.abspath('z_view.png')
+    
+    pl.savefig(x_view,bbox_inches='tight')
+    
+    viz.plot_anat(anat=data, anat_affine=affine, draw_cross=False, slicer='y')
+    pl.savefig(y_view,bbox_inches='tight')
+    
+    viz.plot_anat(anat=data, anat_affine=affine, draw_cross=False, slicer='z')
+    pl.savefig(z_view,bbox_inches='tight')
+    
+    images = [x_view, y_view, z_view]
+    pl.close()
+    return images
+    
+    

fmri/qa/resting_QA.py

@@ -40,6 +40,9 @@ def corr_image(resting_image,fwhm):
     img = nb.load(resting_image)
     corrmat = np.corrcoef(np.squeeze(img.get_data()))
     corrmat[np.isnan(corrmat)] = 0
+    corrmat_npz = os.path.abspath('corrmat.npz')
+    np.savez(corrmat_npz,corrmat=corrmat)
+    
     br = Brain('fsaverage5', 'lh', 'smoothwm')
 
     #br.add_overlay(corrmat[0,:], min=0.2, name=0, visible=True)
@@ -59,7 +62,7 @@ def corr_image(resting_image,fwhm):
 
     #br.add_overlay(precuneus[0,1:], min=0.3, sign='pos', name='mean', visible=True)
 
-    br.add_overlay(precuneus[0,1:], min=0.2, name='mean', visible=True)
+    br.add_overlay(precuneus[0,1:], name='mean', visible=True)
     plt.hist(precuneus[0,1:], 128)
     plt.savefig(os.path.abspath("histogram.png"))
     plt.close()
@@ -80,7 +83,7 @@ def corr_image(resting_image,fwhm):
     roitable=[roitable]
     histogram = os.path.abspath("histogram.png")
 
-    return corr_image, ims, roitable, histogram
+    return corr_image, ims, roitable, histogram, corrmat_npz
 
 def vol2surf(input_volume,ref_volume,reg_file,trg,hemi):
     import os
@@ -149,7 +152,7 @@ def resting_QA(name="resting_QA"):
     workflow.connect(inputspec,'mean_image', tosurf,'ref_volume')
 
     to_img = pe.MapNode(util.Function(input_names=['resting_image','fwhm'],
-                                   output_names=["corr_image","ims","roitable","histogram"],function=corr_image),
+                                   output_names=["corr_image","ims","roitable","histogram","corrmat_npz"],function=corr_image),
                      name="image_gen",iterfield=["resting_image"])
 
     workflow.connect(tosurf,'out_file',to_img,'resting_image')
@@ -161,6 +164,21 @@ def resting_QA(name="resting_QA"):
     sink.inputs.json_sink = c.json_sink
     sink.inputs.Introduction = "Resting state corellations with seed at precuneus"
     sink.inputs.Configuration = start_config_table()
+    
+    corrmat_sinker = pe.Node(nio.DataSink(),name='corrmat_sink')
+    corrmat_sinker.inputs.base_directory = c.json_sink
+    
+    def get_substitutions(subject_id):
+        subs = [('_fwhm_','fwhm'),('_subject_id_%s'%subject_id,'')]
+        for i in range(20):
+            subs.append(('_image_gen%d/corrmat.npz'%i,'corrmat%02d.npz'%i))
+        return subs
+    
+    #corrmat_sinker.inputs.substitutions = #[('_fwhm_','fwhm'),('_subject_id_%s',''),('_image_gen%s','')]
+    workflow.connect(infosource,('subject_id',get_substitutions),corrmat_sinker,'substitutions')
+    workflow.connect(infosource,'subject_id',corrmat_sinker,'container')
+    workflow.connect(to_img,'corrmat_npz',corrmat_sinker,'corrmats')
+    
     #sink.inputs.report_name = "Resting_State_Correlations"
     workflow.connect(infosource,'subject_id',sink,'Subject')
     workflow.connect(fwhmsource,('fwhm',addtitle),sink,'report_name')

fmri/resting/resting_preproc.py

@@ -143,7 +143,7 @@ def prep_workflow(subjects, fieldmap):
     #realign.inputs.loops = 2
     realign.inputs.speedup = 10
     realign.plugin_args = {'qsub_args': '-l nodes=1:ppn=3'}
-
+    preprocess.config = {'execution' : {'crashdump_dir' : c.crash_dir}}
     if len(c.subjects) == 1:
         preprocess.write_graph(graph2use='exec',
                                dotfilename='single_subject_exec.dot')

fmri/resting/sad_resting_config.py

@@ -113,7 +113,10 @@
         'SAD_POST46','SAD_POST47','SAD_POST50','SAD_POST51',
         'SAD_POST52','SAD_POST53','SAD_POST54','SAD_POST56',
         'SAD_POST58']
-subjects = controls+patients+tp2s
+subjects = patients+tp2s
+#['SAD_POST06','SAD2_036','SAD_POST07','SAD_POST46','SAD_023','SAD_018','SAD_POST52','SAD_P57',
+# 'SAD_POST50','SAD_P12','SAD2_022','SAD_POST54','SAD2_050','SAD_P43','SAD_024','SAD_POST10',
+# 'SAD_POST13','SAD_POST05','SAD_POST44','SAD2_044']  #These people had errors
 
 run_on_grid = True
 

fmri/task/first_level.py

@@ -229,6 +229,7 @@ def getsubs(subject_id,getcontrasts,subjectinfo,fwhm):
     c = __import__(fname.split('.')[0])
 
     first_level = combine_wkflw(c)
+    first_level.config = {'execution' : {'crashdump_dir' : c.crash_dir}}
     #first_level.write_graph()
     if run_on_grid:
         first_level.run(plugin='PBS', plugin_args = c.plugin_args)

fmri/task/preproc.py

@@ -143,6 +143,7 @@ def prep_workflow(subjects,fieldmap):
     realign.inputs.speedup = 15
     cc = preprocess.get_node('preproc.CompCor')
     cc.plugin_args = {'qsub_args': '-l nodes=1:ppn=3'}
+    preprocess.config = {'execution' : {'crashdump_dir' : c.crash_dir}}
     if c.run_on_grid:
         preprocess.run(plugin=c.plugin,plugin_args = c.plugin_args)
     else: