From 0cf332b7ea78e8533dc3e5729aff7abc2c9f5bb1 Mon Sep 17 00:00:00 2001 From: Maxime Mulder Date: Wed, 26 Jun 2024 11:21:13 -0400 Subject: [PATCH] rename field --- DTIPrep/DTIPrepRegister.pl | 216 +++---- docs/02-Install.md | 168 ++--- docs/03-TechnicalInfrastructure.md | 344 +++++------ docs/scripts_md/GetRole.md | 56 +- docs/scripts_md/MriScanTypeOB.md | 8 +- docs/scripts_md/minc_to_bids_converter.md | 50 +- docs/scripts_md/register_processed_data.md | 2 +- python/lib/database_lib/files.py | 12 +- python/lib/database_lib/mri_protocol.py | 6 +- .../lib/database_lib/mri_protocol_checks.py | 2 +- python/lib/database_lib/mri_scan_type.py | 8 +- .../nifti_insertion_pipeline.py | 4 +- python/lib/imaging.py | 16 +- python/lib/mri.py | 6 +- tools/delete_imaging_upload.pl | 572 +++++++++--------- tools/get_dicom_files.pl | 110 ++-- tools/minc_to_bids_converter.pl | 96 +-- tools/run_defacing_script.pl | 32 +- uploadNeuroDB/NeuroDB/MRI.pm | 100 ++- uploadNeuroDB/NeuroDB/MRIProcessingUtility.pm | 152 ++--- uploadNeuroDB/NeuroDB/objectBroker/GetRole.pm | 96 +-- .../NeuroDB/objectBroker/MriScanTypeOB.pm | 14 +- .../objectBroker/MriViolationsLogOB.pm | 2 +- uploadNeuroDB/imaging_non_minc_insertion.pl | 2 +- uploadNeuroDB/register_processed_data.pl | 42 +- 25 files changed, 1053 insertions(+), 1063 deletions(-) diff --git a/DTIPrep/DTIPrepRegister.pl b/DTIPrep/DTIPrepRegister.pl index 3e68400ad..2fd2e9cee 100755 --- a/DTIPrep/DTIPrepRegister.pl +++ b/DTIPrep/DTIPrepRegister.pl @@ -97,11 +97,11 @@ =head2 Methods The following output files will be considered: - QCed minc file produced by DTIPrep preprocessing step (i.e. DTI dataset without the bad directions detected by DTIPrep) - QCReport produced by DTPrep - - XMLQCResult produced by DTIPrep + - XMLQCResult produced by DTIPrep - RGB map produced by either DTIPrep or mincdiffusion post-processing (for visually of color artefact) - MD map produced by either DTIPrep or mincdiffusion post-processing - FA map produced by either DTIPrep or mincdiffusion post-processing - - baseline image produced by DTIPrep or mincdiffusion post-processing + - baseline image produced by DTIPrep or mincdiffusion post-processing - DTI mask produced by mincdiffusion post-processing (if mincdiffusion was used to post-process the data) Usage: $0 [options] @@ -218,11 +218,11 @@ =head2 Methods # 1.b Create a hash containing names of all outputs created by DTIPrep/mincdiffusion pipeline my @dti_files = [$dti_file]; # createDTIhashref needs an array with dti_files -my ($DTIrefs) = &DTI::createDTIhashref(@dti_files, - $anat, - $DTIPrep_subdir, - $DTIPrepProtocol, - $protXMLrefs, +my ($DTIrefs) = &DTI::createDTIhashref(@dti_files, + $anat, + $DTIPrep_subdir, + $DTIPrepProtocol, + $protXMLrefs, $QCed2_step ); if (!$DTIrefs) { @@ -246,13 +246,13 @@ =head2 Methods # 2.a Fetch output path stored in $DTIrefs my ($XMLProtocol, $QCReport, $XMLReport, $mri_files) = &getFiles($dti_file, $DTIrefs, $protXMLrefs); -# 2.b Checks that all required outputs were found (there are already a good level of checking in getFiles) +# 2.b Checks that all required outputs were found (there are already a good level of checking in getFiles) # If $QCed_minc is not defined, means that getFiles returned undef for all output files => ERROR. if (!$QCReport) { print LOG "\nERROR:\n\tSome process files are missing in $DTIPrep_subdir.\n"; exit $NeuroDB::ExitCodes::MISSING_FILES; } -# If $QCed2_step is set, QCed2_minc should be defined in %mri_files hash! +# If $QCed2_step is set, QCed2_minc should be defined in %mri_files hash! if (($QCed2_step) && (!$mri_files->{'Preproc'}{'QCed2'}{'minc'})) { print LOG "\nERROR:\n\tSecondary QCed DTIPrep nrrd & minc outputs are " . "missing in $DTIPrep_subdir.\n"; @@ -268,7 +268,7 @@ =head2 Methods ####################### # $registeredXMLprotocolID will store the ID to the registered XMLprotocolFile -my ($registeredXMLprotocolID) = ®ister_XMLProt($XMLProtocol, +my ($registeredXMLprotocolID) = ®ister_XMLProt($XMLProtocol, $data_dir, "DTIPrep"); if (!$registeredXMLprotocolID) { @@ -276,7 +276,7 @@ =head2 Methods exit $NeuroDB::ExitCodes::INSERT_FAILURE; } else { print LOG "\nRegistered XML protocol with ID $registeredXMLprotocolID.\n"; - $mri_files->{'Preproc'}{'QCProt'}{'xml'} = $registeredXMLprotocolID; + $mri_files->{'Preproc'}{'QCProt'}{'xml'} = $registeredXMLprotocolID; } @@ -288,10 +288,10 @@ =head2 Methods my ($report_input_file) = $DTIrefs->{$dti_file}->{'Preproc'}->{'QCReport'}->{'inputs'}->{'Raw_DWI'}; $report_input_file =~ s/$data_dir\///; my ($report_input_fileID) = &getFileID($XMLReport, $report_input_file); -my ($registeredXMLReportFile) = ®ister_XMLFile($XMLReport, - $dti_file, +my ($registeredXMLReportFile) = ®ister_XMLFile($XMLReport, + $dti_file, $data_dir, - $QCReport, + $QCReport, $report_input_fileID, $registeredXMLprotocolID, "DTIPrepPipeline", @@ -312,8 +312,8 @@ =head2 Methods ####################### # $registeredQCReportFile will store the path to the registered QCReportFile -my ($registeredQCReportFile) = ®ister_QCReport($QCReport, - $dti_file, +my ($registeredQCReportFile) = ®ister_QCReport($QCReport, + $dti_file, $data_dir, $report_input_fileID, $registeredXMLprotocolID, @@ -344,7 +344,7 @@ =head2 Methods $registeredXMLprotocolID, "DTIPrepPipeline", $DTIPrepVersion, - 'Preproc', + 'Preproc', 'QCed2' ); $mri_files->{'Preproc'}{'QCed2'}{'minc'} = $QCed2_registered; @@ -357,7 +357,7 @@ =head2 Methods $registeredXMLprotocolID, "DTIPrepPipeline", $DTIPrepVersion, - 'Preproc', + 'Preproc', 'QCed' ); $mri_files->{'Preproc'}{'QCed'}{'minc'} = $QCed_registered; @@ -369,7 +369,7 @@ =head2 Methods ####### Step 7: ####### Register post processed files ####################### -# If mincdiffusion tools were used to create post processed files, register $RGB_minc, $FA_minc, $MD_minc, $baseline_minc, $brain_mask_minc files into the database +# If mincdiffusion tools were used to create post processed files, register $RGB_minc, $FA_minc, $MD_minc, $baseline_minc, $brain_mask_minc files into the database my ($toolName); if ($mri_files->{'Postproc'}{'Tool'} eq "mincdiffusion") { $toolName = $mincdiffVersion; @@ -377,12 +377,12 @@ =head2 Methods $toolName = $DTIPrepVersion; } -my ($postproc_registered, - $postproc_failed_to_register) = ®ister_images($mri_files, +my ($postproc_registered, + $postproc_failed_to_register) = ®ister_images($mri_files, $dti_file, $data_dir, "DTIPrepPipeline", - $toolName, + $toolName, 'Postproc' ); @@ -423,8 +423,8 @@ =head3 register_XMLProt($XMLProtocol, $data_dir, $tool) sub register_XMLProt { my ($XMLProtocol, $data_dir, $tool) = @_; - - my $md5_check = `md5sum $XMLProtocol`; + + my $md5_check = `md5sum $XMLProtocol`; my ($md5sum, $file) = split(' ', $md5_check); my ($ProtocolID) = &fetchProtocolID($md5sum); @@ -435,7 +435,7 @@ sub register_XMLProt { ($ProtocolID) = ®isterProtocol($XMLProtocol, $md5sum, $tool, $data_dir); } - return ($ProtocolID); + return ($ProtocolID); } @@ -457,7 +457,7 @@ =head3 registerProtocol($protocol, $md5sum, $tool, $data_dir) =cut sub registerProtocol { - my ($protocol, $md5sum, $tool, $data_dir) = @_; + my ($protocol, $md5sum, $tool, $data_dir) = @_; # Move file into protocol folder my $tooldir = $data_dir . "/protocols/" . $tool; @@ -508,7 +508,7 @@ sub fetchProtocolID { my $protocolID; if ($sth->rows > 0) { my $row = $sth->fetchrow_hashref(); - $protocolID = $row->{'ProcessProtocolID'}; + $protocolID = $row->{'ProcessProtocolID'}; } return ($protocolID); @@ -548,8 +548,8 @@ sub register_minc { # Determine source file name and source file ID my ($src_name) = basename($raw_file, '.mnc'); my ($src_fileID) = &getFileID($minc, $src_name); - - # Determine pipeline used to create processed data + + # Determine pipeline used to create processed data my ($src_pipeline, $src_tool, $pipelineName_insert, $toolName_insert); if (!$toolName) { # need to develop this later once DTIPrep versioning will be reported into QC reports. @@ -566,21 +566,21 @@ sub register_minc { ); return undef if ((!$toolName_insert) && (!$pipelineName_insert)); } - + # Determine date at which pipeline was run based on registered QC report file my ($pipelineDate) = &getPipelineDate($minc, $data_dir, $registeredQCReportFile); # if date not in $minc, use QC report and insert it into the mincheader. - + # Insert into the mincheader the QC reports (txt & xml) - my ($Txtreport_insert, - $XMLreport_insert) = &insertReports($minc, - $registeredXMLFile, + my ($Txtreport_insert, + $XMLreport_insert) = &insertReports($minc, + $registeredXMLFile, $registeredQCReportFile ); # Insert pipeline summary (how many rejected directions...) into the mincheader - my ($summary_insert) = &insertPipelineSummary($minc, - $data_dir, - $registeredXMLFile, + my ($summary_insert) = &insertPipelineSummary($minc, + $data_dir, + $registeredXMLFile, $scanType); # Insert into the mincheader processed directory of the minc to register @@ -591,7 +591,7 @@ sub register_minc { # Insert nrrd file into minc file if $registered_nrrd is defined my ($nrrd_insert) = &DTI::modify_header('processing:nrrd_file', $registered_nrrd, $minc) if ($registered_nrrd); - + # Determine coordinate '}->{'pace my ($coordinateSpace); $coordinateSpace = "native" if ($toolName =~ /DTIPrep/i); @@ -601,31 +601,31 @@ sub register_minc { my $outputType = "qc"; # Check is all information was correctly inserted into the minc file - return undef unless (($Txtreport_insert) && ($XMLreport_insert) - && ($summary_insert) && ($toolName_insert) + return undef unless (($Txtreport_insert) && ($XMLreport_insert) + && ($summary_insert) && ($toolName_insert) && ($procdir_insert)); # Return undef if a nrrd file was registered but not inserted into the mincheader of the associated minc return undef if (($registered_nrrd) && (!$nrrd_insert)); # If all necessary information are defined, register the file. Return undef otherwise - if (($minc) && ($src_fileID) && - ($src_pipeline) && ($pipelineDate) && - ($coordinateSpace) && ($scanType) && + if (($minc) && ($src_fileID) && + ($src_pipeline) && ($pipelineDate) && + ($coordinateSpace) && ($scanType) && ($outputType) && ($inputs) && ($registeredXMLprotocolID)) { - my ($registeredMincFile) = ®isterFile($minc, - $src_fileID, - $src_pipeline, + my ($registeredMincFile) = ®isterFile($minc, + $src_fileID, + $src_pipeline, $src_tool, - $pipelineDate, - $coordinateSpace, - $scanType, + $pipelineDate, + $coordinateSpace, + $scanType, $outputType, $inputs, $registeredXMLprotocolID - ); - + ); + return ($registeredMincFile); } else { @@ -639,9 +639,9 @@ sub register_minc { "outputType: $outputType\n"; return undef; - } + } -} +} =pod @@ -730,7 +730,7 @@ sub register_XMLFile { return undef; } -} +} =pod @@ -799,21 +799,21 @@ sub register_QCReport { ); return ($registeredQCReportFile); - + } else { - + print LOG "\nERROR: a required option for register_processed_data.pl is not set!!\n"; print LOG "sourceFileID: $src_fileID\n" . "sourcePipeline: $src_pipeline\n" . - "pipelineDate: $pipelineDate\n" . + "pipelineDate: $pipelineDate\n" . "coordinateSpace: $coordinateSpace\n" . - "scanType: $scanType\n" . + "scanType: $scanType\n" . "outputType: $outputType\n"; return undef; - + } -} +} =pod @@ -913,8 +913,8 @@ sub checkPreprocessFiles { $mri_files->{'Preproc'}{'QCed2'}{'nrrd'} = $QCed2_nrrd; $mri_files->{'Preproc'}{'QCed2'}{'minc'} = $QCed2_minc; $mri_files->{'Preproc'}{'QCed2'}{'scanType'} = 'DTIPrepNoReg'; - $mri_files->{'Preproc'}{'QCed'}{'inputs'} = $DTIrefs->{$dti_file}->{'Preproc'}->{'QCed'}->{'inputs'}; - $mri_files->{'Preproc'}{'QCed2'}{'inputs'} = $DTIrefs->{$dti_file}->{'Preproc'}->{'QCed2'}->{'inputs'}; + $mri_files->{'Preproc'}{'QCed'}{'inputs'} = $DTIrefs->{$dti_file}->{'Preproc'}->{'QCed'}->{'inputs'}; + $mri_files->{'Preproc'}{'QCed2'}{'inputs'} = $DTIrefs->{$dti_file}->{'Preproc'}->{'QCed2'}->{'inputs'}; return ($XMLProtocol, $QCReport, $XMLReport); } else { @@ -954,7 +954,7 @@ sub checkPostprocessFiles { my ($dti_file, $DTIrefs, $mri_files) = @_; # Determine file path of each postprocessed outputs common to the two tools (DTIPrep & mincdiffusion) - my $RGB_minc = $DTIrefs->{$dti_file}->{'Postproc'}->{'RGB'}->{'minc'}; + my $RGB_minc = $DTIrefs->{$dti_file}->{'Postproc'}->{'RGB'}->{'minc'}; my $FA_minc = $DTIrefs->{$dti_file}->{'Postproc'}->{'FA'}->{'minc'}; my $MD_minc = $DTIrefs->{$dti_file}->{'Postproc'}->{'MD'}->{'minc'}; my $baseline_minc = $DTIrefs->{$dti_file}->{'Postproc'}->{'baseline'}->{'minc'}; @@ -984,17 +984,17 @@ sub checkPostprocessFiles { my $IDWI_minc = $DTIrefs->{$dti_file}->{'Postproc'}->{'IDWI'}->{'minc'}; my $tensor_minc = $DTIrefs->{$dti_file}->{'Postproc'}->{'tensor'}->{'minc'}; if ((-e $IDWI_minc) && (-e $tensor_minc)) { - $mri_files->{'Postproc'}{'IDWI'}{'minc'} = $IDWI_minc; - $mri_files->{'Postproc'}{'IDWI'}{'scanType'} = 'DTPrepIDWI'; - $mri_files->{'Postproc'}{'tensor'}{'minc'} = $tensor_minc; - $mri_files->{'Postproc'}{'tensor'}{'scanType'} = 'DTIPrepDTI'; + $mri_files->{'Postproc'}{'IDWI'}{'minc'} = $IDWI_minc; + $mri_files->{'Postproc'}{'IDWI'}{'scanType'} = 'DTPrepIDWI'; + $mri_files->{'Postproc'}{'tensor'}{'minc'} = $tensor_minc; + $mri_files->{'Postproc'}{'tensor'}{'scanType'} = 'DTIPrepDTI'; } else { print LOG "Could not find post processing isotropic minc files on the filesystem.\n"; return undef; } # Fetches info about DTIPrep nrrd post processing files - $RGB_nrrd = $DTIrefs->{$dti_file}->{'Postproc'}->{'RGB'}->{'nrrd'}; + $RGB_nrrd = $DTIrefs->{$dti_file}->{'Postproc'}->{'RGB'}->{'nrrd'}; $FA_nrrd = $DTIrefs->{$dti_file}->{'Postproc'}->{'FA'}->{'nrrd'}; $MD_nrrd = $DTIrefs->{$dti_file}->{'Postproc'}->{'MD'}->{'nrrd'}; $baseline_nrrd = $DTIrefs->{$dti_file}->{'Postproc'}->{'baseline'}->{'nrrd'}; @@ -1073,20 +1073,20 @@ sub getFileID { # fetch the FileID of the raw dataset my $query = "SELECT FileID " . - "FROM files " . + "FROM files " . "WHERE File like ?"; - - my $like = "%$src_name%"; - my $sth = $dbh->prepare($query); + + my $like = "%$src_name%"; + my $sth = $dbh->prepare($query); $sth->execute($like); - + if ($sth->rows > 0) { my $row = $sth->fetchrow_hashref(); $fileID = $row->{'FileID'}; }else { print LOG "WARNING: No fileID matches the dataset $src_name used to produce $file.\n\n\n"; } - + return ($fileID); } @@ -1128,7 +1128,7 @@ sub getToolName { $src_tool =~s/"//g; $src_pipeline =~s/"//g; } - + return ($src_pipeline, $src_tool); } @@ -1151,45 +1151,45 @@ =head3 getPipelineDate($file, $data_dir, $QCReport) sub getPipelineDate { my ($file, $data_dir, $QCReport) = @_; - + # Remove $data_dir path from $QCReport in the case it is included in the path $QCReport =~ s/$data_dir//i; my ($pipelineDate, $date_insert); - + if ($file=~/\.mnc/) { $pipelineDate = &DTI::fetch_header_info('processing:processing_date', $file, '$3'); } - + if (!$pipelineDate) { - + print LOG "\n> Fetching date of processing in the QCReport.txt file created by DTIPrep\n"; my $check_line = `cat $data_dir/$QCReport|grep "Check Time"`; $check_line =~s/Check Time://i; # Only keep date info in $check_line. #use Date::Parse library to read the date my ($ss,$mm,$hh,$day,$month,$year,$zone) = strptime($check_line); $pipelineDate = sprintf("%4d%02d%02d", $year+1900, $month+1, $day); - + if ($file=~/\.mnc/) { - # insert pipelineDate into mincheader if not already in the mincheader. + # insert pipelineDate into mincheader if not already in the mincheader. ($date_insert) = &DTI::modify_header( 'processing:processing_date', $pipelineDate, $file ); } - + } else { - + print LOG "\n> Fetching date of processing in the mincheader of $file"; #remove leading spaces, trailing spaces and all instances of " $pipelineDate =~s/"//g; $date_insert = "already inserted"; - + } - + # if date was not inserted into the mincheader, return undef - return undef if (($file=~/\.mnc/) && (!$date_insert)); + return undef if (($file=~/\.mnc/) && (!$date_insert)); # return pipeline date otherwise return ($pipelineDate); } @@ -1253,7 +1253,7 @@ sub insertPipelineSummary { my ($minc, $data_dir, $XMLReport, $scanType) = @_; my ($summary) = &DTI::getRejectedDirections($data_dir, $XMLReport); - + # insert slice wise excluded gradients in mincheader my $rm_slicewise = $summary->{'EXCLUDED'}{'slice'}{'txt'}; my $count_slice = $summary->{'EXCLUDED'}{'slice'}{'nb'}; @@ -1290,7 +1290,7 @@ sub insertPipelineSummary { ); # If all insertions went well, return 1, otherwise return undef - if (($total_insert) && ($insert_slice) && ($insert_inter) + if (($total_insert) && ($insert_slice) && ($insert_inter) && (($insert_gradient) || ($scanType =~ /DTIPrepNoReg/i))) { return 1; } else { @@ -1358,15 +1358,15 @@ sub registerFile { "-protocolID $registeredXMLprotocolID"; system($cmd); print LOG "\n==> Command sent:\n$cmd\n"; - + my ($registeredFile) = &fetchRegisteredFile($src_fileID, $src_pipeline, $pipelineDate, $coordinateSpace, $scanType, $outputType); if (!$registeredFile) { print LOG "> WARNING: No fileID found for SourceFileID=$src_fileID, SourcePipeline=$src_pipeline, PipelineDate=$pipelineDate, CoordinateSpace=$coordinateSpace, ScanType=$scanType and OutputType=$outputType.\n\n\n"; - } + } return ($registeredFile); -} +} =pod @@ -1396,12 +1396,12 @@ sub fetchRegisteredFile { my $query = "SELECT f.File " . "FROM files f " . "JOIN mri_scan_type mst " . - "ON mst.ID=f.AcquisitionProtocolID ". + "ON mst.MriScanTypeID=f.MriScanTypeID ". "WHERE f.SourceFileID=? " . "AND f.SourcePipeline=? " . "AND f.PipelineDate=? " . "AND f.CoordinateSpace=? " . - "AND mst.Scan_type=? " . + "AND mst.MriScanTypeName=? " . "AND OutputType=?"; my $sth = $dbh->prepare($query); @@ -1451,8 +1451,8 @@ sub register_DTIPrep_files { my ($minc, $nrrd, $raw_file, $data_dir, $inputs, $registeredXMLprotocolID, $pipelineName, $DTIPrepVersion, $registeredXMLReportFile, $registeredQCReportFile, $scanType) = @_; # Return undef if variables given as arguments are not defined - return undef unless (($minc) && ($nrrd) - && ($DTIPrepVersion) && ($registeredXMLReportFile) + return undef unless (($minc) && ($nrrd) + && ($DTIPrepVersion) && ($registeredXMLReportFile) && ($registeredQCReportFile) && ($registeredXMLprotocolID) && ($pipelineName) && ($inputs)); @@ -1460,9 +1460,9 @@ sub register_DTIPrep_files { # First checks if DTIPrepReg file exists in DB (could be identical to $noRegQCedDTI) my ($registeredFile, $registeredScanType, $registered_nrrd); if ($scanType eq "DTIPrepReg") { - my $md5_check = `md5sum $nrrd`; + my $md5_check = `md5sum $nrrd`; my ($md5sum, $file) = split(' ', $md5_check); - ($registeredFile, + ($registeredFile, $registeredScanType)= &fetchRegisteredMD5($md5sum); $registered_nrrd = $registeredFile if ($registeredScanType eq 'DTIPrepNoReg'); } @@ -1585,7 +1585,7 @@ sub register_nrrd { return undef; } -} +} =pod @@ -1693,9 +1693,9 @@ sub register_images { $inputs, $registeredXMLprotocolID, $pipelineName, - $toolName, - $registeredXMLReportFile, - $registeredQCReportFile, + $toolName, + $registeredXMLReportFile, + $registeredQCReportFile, $scanType ); @@ -1714,7 +1714,7 @@ sub register_images { ); } - + # Update the minc file in mri_files to the registered_minc; $mri_files->{$process_step}{$proc_file}{'minc'} = $registered_minc; @@ -1722,7 +1722,7 @@ sub register_images { push(@registered, $registered_minc) if ($registered_minc); push(@failed_to_register, $minc) if (!$registered_minc); } - + return (\@registered, \@failed_to_register); } @@ -1749,7 +1749,7 @@ =head3 getInputList($mri_files, $data_dir, $process_step, $proc_file) sub getInputList { my ($mri_files, $data_dir, $process_step, $proc_file) = @_; - my @inputs; + my @inputs; foreach my $input (keys $mri_files->{$process_step}{$proc_file}{'inputs'}) { my $input_file; if ($input =~ m/Raw/i) { @@ -1776,7 +1776,7 @@ sub getInputList { } return ($input_list); -} +} =pod @@ -1796,11 +1796,11 @@ =head3 fetchRegisteredMD5($md5sum) sub fetchRegisteredMD5 { my ($md5sum) = @_; - my $query = "SELECT File, Scan_type" . + my $query = "SELECT File, mst.MriScanTypeName AS ScanType" . " FROM files f" . " JOIN parameter_file pf ON (pf.FileID = f.FileID)" . " JOIN parameter_type pt ON (pt.ParameterTypeID = pf.ParameterTypeID)" . - " JOIN mri_scan_type mst ON (mst.ID=f.AcquisitionProtocolID)" . + " JOIN mri_scan_type mst ON (mst.MriScanTypeID=f.MriScanTypeID)" . " WHERE pt.Name = ? AND pf.Value = ?"; my $sth = $dbh->prepare($query); $sth->execute('md5hash', $md5sum); @@ -1808,7 +1808,7 @@ sub fetchRegisteredMD5 { if ($sth->rows > 0) { my $row = $sth->fetchrow_hashref(); $registeredFile = $row->{'File'}; - $registeredScanType = $row->{'Scan_type'} + $registeredScanType = $row->{'ScanType'} } return ($registeredFile, $registeredScanType); diff --git a/docs/02-Install.md b/docs/02-Install.md index f390025d4..7e31187a3 100644 --- a/docs/02-Install.md +++ b/docs/02-Install.md @@ -8,17 +8,17 @@ Dependencies and installation information are documented on the LORIS-MRI ## 2.2 Configuration -Following a successful install, some configurations and customizations are +Following a successful install, some configurations and customizations are needed and outlined in the next three sub-sections. ### 2.2.1 Database -The following tables in the database need to be configured properly for the +The following tables in the database need to be configured properly for the insertion pipeline to successfully insert scans. 1. **`psc`** table -The `MRI_alias` field must be populated for each site that is scanning +The `MRI_alias` field must be populated for each site that is scanning candidates or phantoms. 2. **`Visit_Windows`** table @@ -28,11 +28,11 @@ To populate with visit labels, you can manually insert study-specific informatio INSERT INTO Visit_Windows (Visit_label, WindowMinDays, WindowMaxDays, OptimumMinDays, OptimumMaxDays, WindowMidpointDays) VALUES ('V1', '0', '100', '40', '60', '50'); -If age is not a critical factor in study visit scheduling, define `Min` value as +If age is not a critical factor in study visit scheduling, define `Min` value as `0`, and `Max` value as `2147483647` (maximum `int`). -Alternatively, LORIS provides a PHP script called `populate_visit_windows.php` +Alternatively, LORIS provides a PHP script called `populate_visit_windows.php` in its `tools/` directory that can be used. @@ -40,7 +40,7 @@ in its `tools/` directory that can be used. > - `mri_scan_type`: this table is a lookup table that stores the name of the acquisition (*e.g.* t1, t2, flair...). Do not include commas, hyphens, -spaces or periods in your `mri_scan_type.Scan_type` column values. The ID +spaces or periods in your `mri_scan_type.MriScanTypeName` column values. The ID present in this table will be used in the `mri_protocol` and `mri_protocol_checks` tables described below. The ID is also used in the `bids_mri_scan_type_rel` table that will be described in 5. @@ -81,7 +81,7 @@ there will be no constraint on that header. 5. `bids_mri_scan_type_rel` and other `bids*` tables -The following tables are used by pipelines generating BIDS files either directly from DICOM files +The following tables are used by pipelines generating BIDS files either directly from DICOM files (using dcm2niix as a converter) or from the MINC files already inserted into the database. > - `bids_mri_scan_type_rel`: this table maps a given scan type with BIDS labelling convention @@ -96,7 +96,7 @@ modality folder where the NIfTI and JSON files will go (examples: `anat`, `func` the NIfTI and JSON files (examples: `T1w`, `T2w`, `bold`, `dwi`, `FLAIR`, `magnitude`, `phasediff`) > - `bids_scan_type_subcategory`: stores the series of entities used to label acquisitions -(may refer to a custom study protocol) within the NIfTI and JSON file names +(may refer to a custom study protocol) within the NIfTI and JSON file names (e.g. `acq-25direction`, `task-rest`, `dir-AP`, `dir-PA`, `acq-B0_dir-AP`). Refer to the [BIDS specification](https://bids-specification.readthedocs.io/en/stable/) for details on how to label acquisitions. @@ -115,25 +115,25 @@ how to properly fill in the BIDS tables above so that it is fully standard-compl 6. **`Config`** table The `Config` table can also be accessed and customized from the LORIS front-end -via the `Configuration` module, accessible under the `Admin` menu. Here are the +via the `Configuration` module, accessible under the `Admin` menu. Here are the configuration settings that impact directly or indirectly the pipeline: Under the `Study` section: * `ImagingUploader Auto Launch`: used by the Imaging Uploader to automatically launch the insertion scripts on the uploaded scan - -Under the `Paths` section: + +Under the `Paths` section: * `LORIS-MRI Code`: where the LORIS-MRI codebase is installed; typically `/opt/$PROJECT/bin/mri/` * `MRI Incoming Directory`: where the uploaded scans get stored; typically `/data/incoming/` * `Images`: where the images displayed in Imaging Browser are stored; typically `/data/$PROJECT/data/` - -Under the `Imaging Modules` section: + +Under the `Imaging Modules` section: * `Patient ID regex`: used by the DICOM Archive module to show/hide the PatientID info * `Patient name regex`: used by the DICOM Archive module to show/hide the Patient Name info * `Lego phantom regex`: used by the DICOM Archive module to show/hide the Patient Name info for phantoms * `Living phantom regex`: used by the DICOM Archive module to show/hide the Patient Name info for phantoms * `Imaging Browser Tabulated Scan Types`: used by Imaging Browser's main page which lists the different imaging sessions across candidates. This setting will determine which modalities will have their QC status displayed in that listing page - -Under the `Imaging Pipeline` section: + +Under the `Imaging Pipeline` section: * `LORIS-MRI Data Directory`: directory where imaging data is stored; typically `/data/$PROJECT/data/` * `Study Name`: prefix to be used in all filenames inserted into the `files` @@ -145,7 +145,7 @@ Under the `Imaging Pipeline` section: should be used by mincpik when generating pictures to be displayed in Imaging Browser * `NIfTI file creation`: used to enable or disable automated NIfTI file creation * `DICOM converter tool to use (dcm2mnc or dcm2niix)`: allows the user to specify the binary - file to be used when converting DICOM files to MINC or NIfTI files. The default setting is to + file to be used when converting DICOM files to MINC or NIfTI files. The default setting is to use the binary provided by the MINC tools, namely `dcm2mnc` for studies wishing to generate MINC files. For studies that want BIDS dataset generated out of the DICOM files, then specify `dcm2niix` * `Path to Tarchives`: directory where the original DICOMs are archived; @@ -178,8 +178,8 @@ Under the `Imaging Pipeline` section: directions that can be removed from a DTI scan to pass QC. Used by the DTIPrep pipeline * `NIAK Path`: Path to NIAK if MINC diffusion is to be run. Used by the DTIPrep pipeline * `Secondary QCed dataset`: path where a secondary QC'ed dataset is to be stored. Used by the DTIPrep pipeline - * `Series description to exclude from imaging insertion`: series descriptions to be - excluded from the steps of the pipeline that start at, and follow the DICOM to + * `Series description to exclude from imaging insertion`: series descriptions to be + excluded from the steps of the pipeline that start at, and follow the DICOM to MINC conversion. Note that the series description entered in that field needs to be an exact match of what is present in the DICOM series description field. * `ComputeDeepQC`: enable or disable the automated computation of image quality @@ -189,18 +189,18 @@ Under the `Imaging Pipeline` section: * `Name of the environment file`: name of the MRI environment file to source before running the insertion scripts; typically `environment`; used when Auto-launch is turned on for the pipeline. - * `Modalities on which SNR should be calculated`: list of modalities/scan types on + * `Modalities on which SNR should be calculated`: list of modalities/scan types on which to compute SNR; typically all 3D images * `Scan type to use as a reference for defacing (typically T1W image)`: scan type name of the modality to use as a reference for defacing * `Modalities on which to run the defacing pipeline`: list of modalities/scan types - on which the defacing algorithm should be run; typically any scan showing the + on which the defacing algorithm should be run; typically any scan showing the face of the candidate * `Name of the Python MRI config file`: name of the python-based MRI config file to use when running the Python insertion scripts (typically `database_config.py`) - -Under the `MINC to BIDS Converter Tool Options` section: - * `BIDS Dataset Authors`: list of authors who should be included in the + +Under the `MINC to BIDS Converter Tool Options` section: + * `BIDS Dataset Authors`: list of authors who should be included in the dataset_description.json BIDS file (generated by this script) * `BIDS Dataset Acknowledgments`: string with acknowledgment information to be used when generating the dataset_description.json BIDS file @@ -214,85 +214,85 @@ to be ignored when running the BIDS Validator on the generated BIDS dataset 1. **Imaging Uploader** -Projects can upload scans and launch the pipeline in a variety of options -detailed in the [PipelineLaunchOptions](05-PipelineLaunchOptions.md) section. -Irrespective of the project's choice as to whether the imaging scan is to be -uploaded through the Imaging Uploader GUI or not, pipeline insertion progress +Projects can upload scans and launch the pipeline in a variety of options +detailed in the [PipelineLaunchOptions](05-PipelineLaunchOptions.md) section. +Irrespective of the project's choice as to whether the imaging scan is to be +uploaded through the Imaging Uploader GUI or not, pipeline insertion progress can be consulted through a live 'Log Viewer' panel. -Some settings need to be configured properly (`php.ini` variables, -`MRI-Upload Directory` and `ImagingUploader Auto Launch`), and are documented in +Some settings need to be configured properly (`php.ini` variables, +`MRI-Upload Directory` and `ImagingUploader Auto Launch`), and are documented in the [LORIS repository: Imaging Uploader Specification](https://github.com/aces/Loris/blob/main/modules/imaging_uploader/README.md). 2. **DICOM Archive** -This LORIS module provides a front-end display with the details of the archived -DICOM study from the database `tarchive_*` tables. The only setting that -impacts the display of this module are the regex settings in the `Configuration` -module under the section `Imaging Modules`. These settings determine whether the -Patient Name/Patient ID header values are displayed in full, or show up as +This LORIS module provides a front-end display with the details of the archived +DICOM study from the database `tarchive_*` tables. The only setting that +impacts the display of this module are the regex settings in the `Configuration` +module under the section `Imaging Modules`. These settings determine whether the +Patient Name/Patient ID header values are displayed in full, or show up as **INVALID-HIDDEN**. -More detailed specifications can be consulted in the +More detailed specifications can be consulted in the [LORIS repository: DICOM Archive Specification](https://github.com/aces/Loris/blob/main/modules/dicom_archive/README.md). 3. **Imaging Browser** -The Imaging Browser module accesses the screenshot (PIC) images directly from -the filesystem where they are stored. It also provides the option to download +The Imaging Browser module accesses the screenshot (PIC) images directly from +the filesystem where they are stored. It also provides the option to download some files. Ensure that: - `/data/$PROJECT` directory and subdirectories are readable and executable by the Apache linux user. -- the Configuration module (*Paths*) `Images` setting is set (typically: `/data/$PROJECT/data/`). - -More detailed specifications can be consulted in the +- the Configuration module (*Paths*) `Images` setting is set (typically: `/data/$PROJECT/data/`). + +More detailed specifications can be consulted in the [LORIS repository: Imaging Browser Specification](https://github.com/aces/Loris/blob/main/modules/imaging_browser/README.md). 4. **Brainbrowser** -Brainbrowser displays the MINC (or NIfTI) images within the browser. It accesses those +Brainbrowser displays the MINC (or NIfTI) images within the browser. It accesses those images directly from the filesystem. Ensure that: - `/data/$PROJECT` directory and subdirectories are readable and executable by the Apache linux user. - the Configuration module (*Paths*) `Images` setting is `/data/$PROJECT/data/`. - the `project/config.xml` file (in the main LORIS codebase) contains the proper MINC toolkit path in the `` tagset. - -More detailed specifications can be consulted in the + +More detailed specifications can be consulted in the [LORIS repository: Brainbrowser Specification](https://github.com/aces/Loris/blob/main/modules/brainbrowser/README.md). 5. **MRI Violated Scans** -No configuration setting is needed for the MRI Violated Scans module to work. -Data loaded in this module gets populated automatically by the insertion -scripts. As such, scans whose parameters can't be matched against the -`mri_protocol` table during the imaging insertion process, will be flagged as -protocol violations and will not have their MINC/NIfTI volumes loaded in the -Imaging Browser module. Violated scans can be viewed and the type of error +No configuration setting is needed for the MRI Violated Scans module to work. +Data loaded in this module gets populated automatically by the insertion +scripts. As such, scans whose parameters can't be matched against the +`mri_protocol` table during the imaging insertion process, will be flagged as +protocol violations and will not have their MINC/NIfTI volumes loaded in the +Imaging Browser module. Violated scans can be viewed and the type of error (scan identification, protocol violation) can be reviewed from the front-end. -More detailed specifications can be consulted in the +More detailed specifications can be consulted in the [LORIS repository: MRI Violated Scans Specification](https://github.com/aces/Loris/blob/main/modules/mri_violations/README.md). 6. **Electrophysiology Browser** -No configuration setting is needed for the Electrophysiology Browser module. -Data loaded in this module get populated automatically by the BIDS insertion -scripts (in the `python` directory). It accesses data stored in the +No configuration setting is needed for the Electrophysiology Browser module. +Data loaded in this module get populated automatically by the BIDS insertion +scripts (in the `python` directory). It accesses data stored in the `physiological_*` tables. -### 2.2.3 LORIS-MRI +### 2.2.3 LORIS-MRI #### Filesystem - `/data/*` and `/opt/*` subdirectories were created by the imaging install script. If not, it may be due to `root:root` ownership of the `/data/` mount or `/opt` directory on your system. Ensure these subdirectories are created manually, particularly: - `/opt/$PROJECT/bin/mri/`, `/data/incoming/`, and those inside + `/opt/$PROJECT/bin/mri/`, `/data/incoming/`, and those inside `/data/$PROJECT/data/`, namely `assembly`, `assembly_bids`, `batch_output`, `bids_imports`, `logs`, `pic`, `tarchive`, and `trashbin`. @@ -300,15 +300,15 @@ scripts (in the `python` directory). It accesses data stored in the - `/data/$PROJECT/` and `/opt/$PROJECT/` directory and subdirectories must be readable and executable by the Apache linux user. It may also help to ensure the `/data/` and `/opt/` mount is executable. After any modifications, ensure you restart apache. - + #### Customizable routines in the Perl config file (a.k.a. `prod` under `dicom-archive/.loris_mri`) - `isFileToBeRegisteredGivenProtocol()` - * By default, any scan will be inserted if it matches an `mri_protocol` + * By default, any scan will be inserted if it matches an `mri_protocol` table entry. - * To **whitelist/blacklist** specific scan types -- *e.g.* in the case of - protocol exclusion, case sensitivity or labelling variance -- modify the + * To **whitelist/blacklist** specific scan types -- *e.g.* in the case of + protocol exclusion, case sensitivity or labelling variance -- modify the subroutine, *e.g.*: ```perl @@ -317,13 +317,13 @@ if($acquisitionProtocol eq 't1' or $acquisitionProtocol eq 't2' or $acquisitionP - `getSubjectIDs()` - Routine to parse candidate’s PSCID, CandID, Center (determined from the PSCID), and visit - label. - + Routine to parse candidate’s PSCID, CandID, Center (determined from the PSCID), and visit + label. + - `get_DTI_CandID_Visit()` Used by the DTIPrep pipeline - + - `determineHRRTprotocol()` Routine to determine the acquisition protocol to use to register an HRRT derived @@ -380,7 +380,7 @@ The following must be recursively owned by the `lorisadmin` user and Apache grou #### 2.3.4 Verify Configuration module settings for Imaging Pipeline -In the LORIS front-end, under the Admin menu, go to the `Config` module. Verify/set +In the LORIS front-end, under the Admin menu, go to the `Config` module. Verify/set the following config settings (examples are shown below for a project named `demo`): Under the `Imaging Pipeline` section: @@ -390,7 +390,7 @@ Under the `Imaging Pipeline` section: * `Full path to get_dicom_info.pl script`(typically `/opt/$PROJECT/bin/mri/dicom-archive/get_dicom_info.pl`) * `Path to Tarchives` (typically `/data/$PROJECT/data/tarchive/`) * `Default visit label for BIDS dataset`: (`V01` or any visit label fitting) - * `DICOM converter tool to use (dcm2mnc or dcm2niix)`: must be used to specify which tool the pipeline + * `DICOM converter tool to use (dcm2mnc or dcm2niix)`: must be used to specify which tool the pipeline should run -- `dcm2mnc` to produce MINC files, or `dcm2niix` to produce BIDS-compatible files Under the `Path` section: @@ -421,19 +421,19 @@ Typically, images insertion into LORIS is performed via the following steps: hosting LORIS). 2. DICOM insertion into the `tarchive` tables in order to be able to see the information stored in the DICOMs via the **DICOM Archive** module. -3. Conversion of the DICOMs into MINC files (via dcm2mnc) or BIDS NIfTI and JSON +3. Conversion of the DICOMs into MINC files (via dcm2mnc) or BIDS NIfTI and JSON files (via dcm2niix). Those converted files undergo protocol validation and insertion into the LORIS database. By default, NIfTI images will be generated from the MINC files and inserted into LORIS when generating MINC files (see the [Configuration](#2.2-configuration) section above if you want to disable this option). One of two possible actions will follow depending on the study-defined protocol: - + a. If a scan matches one of the protocol defined in the `mri_protocol` - table and passes the optional additional file checks present in the - `mri_protocol_checks` table, then the image will be stored into - the `files` tables. This inserted image is then accessible via the - **Imaging Browser** module and can be displayed in 3D using + table and passes the optional additional file checks present in the + `mri_protocol_checks` table, then the image will be stored into + the `files` tables. This inserted image is then accessible via the + **Imaging Browser** module and can be displayed in 3D using **BrainBrowser**. b. If a scan does not match any of the protocol defined in the @@ -458,13 +458,13 @@ The graph below shows the different modules mentioned above with the ### 2.4.2 PET data from an HRRT scanner -The pipeline was initially designed for **raw HRRT PET datasets collected at +The pipeline was initially designed for **raw HRRT PET datasets collected at the Brain Imaging Center of the Montreal Neurological Institute**. Since there is no standard for HRRT datasets and only 7 scanners existing in the world, the insertion pipeline of PET data from an HRRT scanner might need to be updated/modified for other scanners and will be done when the need comes. -Image insertion of PET dataset from an HRRT scanner is very similar to the +Image insertion of PET dataset from an HRRT scanner is very similar to the insertion described for DICOM datasets, to the exception that the HRRT archive information are stored in the `hrrt_archive` tables instead of the `tarchive` tables. @@ -476,28 +476,28 @@ Insertion into LORIS is performed via the following steps: 2. HRRT insertion into the `hrrt_archive` tables. 3. Conversion of the ECAT7 files into MINC files for protocol identification and insertion into the LORIS database. Note that the ECAT7 images will be linked - to the inserted MINC files. - + to the inserted MINC files. + ### 2.4.3 BIDS insertion (Electrophysiology and Imaging) -The pipeline to insert BIDS datasets into LORIS currently support the +The pipeline to insert BIDS datasets into LORIS currently support the following BIDS modalities/entities: - Electroencephalography ('eeg') - - Magnetic Resonance Imaging ('anat', 'func', 'fmap', 'dwi') + - Magnetic Resonance Imaging ('anat', 'func', 'fmap', 'dwi') - Intracranial electroencephalography ('ieeg') - - **Note:** optional [electrical stimulation](https://bids-specification.readthedocs.io/en/stable/04-modality-specific-files/04-intracranial-electroencephalography.html#electrical-stimulation) files for ieeg modality are currently not supported in LORIS-MRI (or LORIS). - If a BIDS-IEEG directory includes these files, they will be ignored during the import process. + - **Note:** optional [electrical stimulation](https://bids-specification.readthedocs.io/en/stable/04-modality-specific-files/04-intracranial-electroencephalography.html#electrical-stimulation) files for ieeg modality are currently not supported in LORIS-MRI (or LORIS). + If a BIDS-IEEG directory includes these files, they will be ignored during the import process. -With slight modifications and further customization, it could handle other +With slight modifications and further customization, it could handle other types of electrophysiology or imaging modalities. Typically, BIDS data insertion into LORIS is performed via the following steps: -1. Transfer the BIDS-format data to the LORIS server via commandline. Due to the -large size of electrophysiological/BIDS data, a suitable browser-based uploader is +1. Transfer the BIDS-format data to the LORIS server via commandline. Due to the +large size of electrophysiological/BIDS data, a suitable browser-based uploader is not presently available. -2. Run the BIDS import script to import the data into the `physiological_*` tables -for electrophysiology datasets and into the `file*` tables of LORIS. More details about this import script can +2. Run the BIDS import script to import the data into the `physiological_*` tables +for electrophysiology datasets and into the `file*` tables of LORIS. More details about this import script can be found in the [PipelineLaunchOptions](05-PipelineLaunchOptions.md) section. diff --git a/docs/03-TechnicalInfrastructure.md b/docs/03-TechnicalInfrastructure.md index 7cba21c9c..db1ff5c79 100644 --- a/docs/03-TechnicalInfrastructure.md +++ b/docs/03-TechnicalInfrastructure.md @@ -39,31 +39,31 @@ The imaging part of a LORIS instance is typically separated into two directories |__ protocols* ``` -`*` _denotes optional directories that are not automatically created by the +`*` _denotes optional directories that are not automatically created by the install script. They are created when running the `DTIprep` pipeline_ Within a LORIS-MRI instance there are typically two directories: - The `/opt/$PROJECT/bin/mri` directory is a copy of all the imaging scripts downloaded from - the [GitHub LORIS-MRI repository](https://github.com/aces/Loris-MRI). + the [GitHub LORIS-MRI repository](https://github.com/aces/Loris-MRI). Details about the content of this folder can be found in the [script section](04-Scripts.md). - The `/data/$PROJECT/data` directory stores all the imaging-related data that will be created - by the imaging scripts. + by the imaging scripts. -The following subsections will describe the content of the different +The following subsections will describe the content of the different subdirectories found under `/data/$PROJECT/data`. #### The `assembly` directory -The MINC images that can be viewed via BrainBrowser in the imaging browser - module are located under the `data/assembly` directory and organized by - `CandID/Visit`. Within each of these visit labels, data are first - organized by imaging type (`mri` or `pet` for example) and then by output - type (such as `native` or `processed`). For example, a native T1W image for - subject 123456's V1 visit will be located in +The MINC images that can be viewed via BrainBrowser in the imaging browser + module are located under the `data/assembly` directory and organized by + `CandID/Visit`. Within each of these visit labels, data are first + organized by imaging type (`mri` or `pet` for example) and then by output + type (such as `native` or `processed`). For example, a native T1W image for + subject 123456's V1 visit will be located in `data/assembly/123456/V1/mri/native/project_123456_V1_T1W_001.mnc`. ``` @@ -85,10 +85,10 @@ The MINC images that can be viewed via BrainBrowser in the imaging browser #### The `assembly_bids` directory -The BIDS images derived from DICOM files that can be viewed via BrainBrowser in the Imaging Browser - module are located under the `data/assembly_bids` directory and organized as - a BIDS structure (see BIDS [specifications](https://bids.neuroimaging.io/)). - For example, a native T1W image for subject 123456's V1 visit will be located in +The BIDS images derived from DICOM files that can be viewed via BrainBrowser in the Imaging Browser + module are located under the `data/assembly_bids` directory and organized as + a BIDS structure (see BIDS [specifications](https://bids.neuroimaging.io/)). + For example, a native T1W image for subject 123456's V1 visit will be located in `data/assembly_bids/sub-123456/ses-V1/anat/sub-123456_ses-V1_run-1_T1w.nii.gz` along with its JSON side car file `data/assembly_bids/sub-123456/ses-V1/anat/sub-123456_ses-V1_run-1_T1w.json`. @@ -110,34 +110,34 @@ The BIDS images derived from DICOM files that can be viewed via BrainBrowser in #### The `incoming` directory -Incoming scans from the Imaging uploader module (or automatic cron jobs) are +Incoming scans from the Imaging uploader module (or automatic cron jobs) are stored in an `incoming` directory. Once the pipeline has successfully run, data in the incoming folder are removed to avoid duplication of raw imaging datasets. - - + + #### The `bids_imports` directory -Every BIDS data structure imported into LORIS will be stored in the -`bids_imports` directory. Within that directory, each imported BIDS structure -will be contained into a sub-directory that will be named based on the +Every BIDS data structure imported into LORIS will be stored in the +`bids_imports` directory. Within that directory, each imported BIDS structure +will be contained into a sub-directory that will be named based on the following information: -- the "Name" field of the `dataset_description.json` BIDS file (for example +- the "Name" field of the `dataset_description.json` BIDS file (for example "Example BIDS Dataset") -- the "BIDSVersion" field of the `dataset_description.json` BIDS file (for +- the "BIDSVersion" field of the `dataset_description.json` BIDS file (for example "1.0.2") -For the example mentioned here, the sub-directory would be named: -`Example_BIDS_Dataset_BIDSVersion_1.0.2` (Note how spaces present in the +For the example mentioned here, the sub-directory would be named: +`Example_BIDS_Dataset_BIDSVersion_1.0.2` (Note how spaces present in the "Name" field were replaced by underscores in the sub-directory name) -Within that BIDS sub-directory, the file system structure will follow the +Within that BIDS sub-directory, the file system structure will follow the [BIDS specification][1]. #### The `hrrtarchive` directory - + The HRRT archives listed in the `hrrt_archive` table are stored in the - `data/hrrtarchive` directory and organized within folders representing the + `data/hrrtarchive` directory and organized within folders representing the different years of acquisition. ``` @@ -152,12 +152,12 @@ The HRRT archives listed in the `hrrt_archive` table are stored in the |__ DCM_`date`_hrrtarchive.tar |__ DCM_`date`_hrrtarchive.tar ``` - - + + #### The `logs` directory The logs of the scripts are created under `data/logs` in `/data/$PROJECT`. - + ``` ## Content of the /data/$PROJECT/data/logs directory . @@ -173,32 +173,32 @@ The logs of the scripts are created under `data/logs` in `/data/$PROJECT`. |__ registerProcessed`date`.log ``` -`*` _denotes optional directories that are not automatically created by the +`*` _denotes optional directories that are not automatically created by the install script_ -`hh-mm-xxxxxx` _where_ `hh` _denotes the hour the insertion to the database +`hh-mm-xxxxxx` _where_ `hh` _denotes the hour the insertion to the database took place,_ `mm` _the minutes, and_ `xxxxxx` _a random alphanumeric string._ #### The `pic` directory -The screenshots displayed in the imaging browser module for each modality is - stored within the `data/pic` folder and organized per candidates. - +The screenshots displayed in the imaging browser module for each modality is + stored within the `data/pic` folder and organized per candidates. + ``` ## Content of the /data/$PROJECT/data/pic directory . |__ CandID |__ project_CandID_Visit_modality_number_fileid_check.jpg |__ project_CandID_Visit_modality_number_fileid_check.jpg -``` +``` #### The `pipelines` directory -Processed incoming data or DTIPrep pipeline outputs are stored within - the `data/pipelines` directory and organized per pipeline versions, - candidates and visit labels. In addition, protocol files for automatic +Processed incoming data or DTIPrep pipeline outputs are stored within + the `data/pipelines` directory and organized per pipeline versions, + candidates and visit labels. In addition, protocol files for automatic pipelines are saved in the `data/protocols` directory. - + ``` ## Content of the /data/$PROJECT/data/pipelines directory . @@ -221,9 +221,9 @@ Processed incoming data or DTIPrep pipeline outputs are stored within #### The `tarchive` directory - + The DICOM archives listed in the DICOM archive module are stored in the - `data/tarchive` directory and organized within folders representing the + `data/tarchive` directory and organized within folders representing the different years of acquisition. ``` @@ -241,9 +241,9 @@ The DICOM archives listed in the DICOM archive module are stored in the #### The `trashbin` directory -The scans that violates the established imaging protocol and listed in the MRI +The scans that violates the established imaging protocol and listed in the MRI violated scans module are stored within the directory `data/trashbin`. - + ``` ## Content of the /data/$PROJECT/data/trashbin directory . @@ -253,13 +253,13 @@ The scans that violates the established imaging protocol and listed in the MRI |__ Tarload-XX2 |__file.mnc ``` - + ## 3.2 Database infrastructure -The database infrastructure is divided in six main components based on the - workflow happening from native images insertion to quality control and - ultimately insertion of processed datasets. +The database infrastructure is divided in six main components based on the + workflow happening from native images insertion to quality control and + ultimately insertion of processed datasets. ![overall_DB_structure](images/overall_DB_structure.png) @@ -268,10 +268,10 @@ The database infrastructure is divided in six main components based on the #### 3.2.1.1 MRI upload table -Summary information about the imaging upload status can be found in the - `mri_upload` table. This includes links to the DICOM archive tables (described - in the next section) and to the `session` table. It also includes summary - information regarding the upload and the insertion process performed after +Summary information about the imaging upload status can be found in the + `mri_upload` table. This includes links to the DICOM archive tables (described + in the next section) and to the `session` table. It also includes summary + information regarding the upload and the insertion process performed after the upload. ![mri_upload_tables](images/mri_upload_tables.png) @@ -279,91 +279,91 @@ Summary information about the imaging upload status can be found in the #### 3.2.1.2 Tarchive tables -The first step to insert a new imaging session into the database is the +The first step to insert a new imaging session into the database is the insertion of the DICOM study. In the database, all information related to a DICOM study is being organized into three different tables: - + * the `tarchive` table stores information about the whole imaging session, - including patient, scanner and study information, as well as the location - of the archived DICOM dataset. Each row correspond to a specific imaging + including patient, scanner and study information, as well as the location + of the archived DICOM dataset. Each row correspond to a specific imaging session identified by the DICOM header `StudyUID`. - * the `tarchive_series` table stores information about each modality that - was acquired during the imaging session (T1W, T2W...). This information - include imaging parameters such as TR, TE, TI, slice thickness, sequence - name... Each row corresponds to a different modality identified by the - DICOM header `SeriesUID` and `EchoTime`. This table is linked to the + * the `tarchive_series` table stores information about each modality that + was acquired during the imaging session (T1W, T2W...). This information + include imaging parameters such as TR, TE, TI, slice thickness, sequence + name... Each row corresponds to a different modality identified by the + DICOM header `SeriesUID` and `EchoTime`. This table is linked to the `tarchive` table via the `TarchiveID` foreign key. - * the `tarchive_files` table stores information about each DICOM found in - the imaging session. Each row correspond to one DICOM file and is linked - to the `tarchive` table via the `TarchiveID` foreign key and to the + * the `tarchive_files` table stores information about each DICOM found in + the imaging session. Each row correspond to one DICOM file and is linked + to the `tarchive` table via the `TarchiveID` foreign key and to the `tarchive_series` table via the `TarchiveSeriesID` foreign key. - + ![tarchive_tables](images/tarchive_tables.png) -In the front end of LORIS, you can see the DICOM studies using the - _DICOM Archive_ module under the _Imaging_ tab. The information displayed in +In the front end of LORIS, you can see the DICOM studies using the + _DICOM Archive_ module under the _Imaging_ tab. The information displayed in this module comes from the three `tarchive` tables mentioned above. -Note: the `SessionID` field of the `tarchive` table is populated once at least - one MINC file derived from that DICOM study got inserted in the tables +Note: the `SessionID` field of the `tarchive` table is populated once at least + one MINC file derived from that DICOM study got inserted in the tables described in 3.2.1.3. - + #### 3.2.1.2 HRRT archive tables -The first step to insert a new HRRT session into the database is the +The first step to insert a new HRRT session into the database is the insertion of the HRRT PET study. In the database, all information related to a HRRT PET study is being organized into two different tables: - + * the `hrrt_archive` table stores information about the whole HRRT session, - including patient, center name and study information, as well as the location - of the archived HRRT dataset. Each row correspond to a specific HRRT + including patient, center name and study information, as well as the location + of the archived HRRT dataset. Each row correspond to a specific HRRT session. - * the `hrrt_archive_files` table stores information about each ECAT7 file found in - the HRRT session. Each row correspond to one ECAT7 file and is linked + * the `hrrt_archive_files` table stores information about each ECAT7 file found in + the HRRT session. Each row correspond to one ECAT7 file and is linked to the `hrrt_archive` table via the `HrrtArchiveID` foreign key. -Note: the `SessionID` field of the `hrrt_archive` table is populated once at least - one MINC file derived from that HRRT study got inserted in the tables +Note: the `SessionID` field of the `hrrt_archive` table is populated once at least + one MINC file derived from that HRRT study got inserted in the tables described in 3.2.1.3. #### 3.2.1.3 Files tables -The second step to insert a new imaging session into the database is the - conversion of the DICOM study into the MINC files that will be inserted based - on the imaging protocol used. Having the dataset converted in MINC allows +The second step to insert a new imaging session into the database is the + conversion of the DICOM study into the MINC files that will be inserted based + on the imaging protocol used. Having the dataset converted in MINC allows visualization of the images directly in the browser. -First, all DICOMs are converted into either MINC format (using `dcm2mnc` - from MINC tools), or into a BIDS structure (using `dcm2niix`). Then, - the backend scripts will pull the information stored in the following tables +First, all DICOMs are converted into either MINC format (using `dcm2mnc` + from MINC tools), or into a BIDS structure (using `dcm2niix`). Then, + the backend scripts will pull the information stored in the following tables in order to identify the scan type for each converted file created: - - * the `mri_scan_type` table stores the name of the scan type linked - along with the `ID` field that will be used to identify the scan type - * the `mri_protocol` table stores each scan type's parameters that will + + * the `mri_scan_type` table stores the name of the scan type linked + along with the `MriScanTypeID` field that will be used to identify the scan type + * the `mri_protocol` table stores each scan type's parameters that will be used to identify the scan type (TR, TE, TI, slice_thickness...) * the `mri_protocol_group` defines the groups of scanning protocols. Each line - in the `mri_protocol` table belongs to one and only one group. + in the `mri_protocol` table belongs to one and only one group. * the `mri_protocol_group_target` table is used to determine which scanning protocol - group to use in order to identify the type of a given scan based on the + group to use in order to identify the type of a given scan based on the subject's project, the subject's cohort or the visit at which the scan was done. - * the `mri_protocol_checks` table stores additional protocol checks - after an acquisition has been identified in order to automatically flag + * the `mri_protocol_checks` table stores additional protocol checks + after an acquisition has been identified in order to automatically flag some acquisitions based on information stored in specific DICOM headers * the `mri_protocol_checks_group` defines the different groups of protocol checks. Each lines in the `mri_protocol_checks` table belongs to one and only one group. * the `mri_protocol_checks_group_target` table is used to determine which protocol checks to use when an archive is processed by the MRI pipeline based on the - subject's project, the subject's cohort or the visit at which the scan was + subject's project, the subject's cohort or the visit at which the scan was done. - * the `bids_mri_scan_type_rel*` table is used to determine the name convention to be - used for the BIDS data based on a specific scan type. It is linked to the - `mri_scan_type`, `bids_category`, `bids_phase_encoding_direction`, + * the `bids_mri_scan_type_rel*` table is used to determine the name convention to be + used for the BIDS data based on a specific scan type. It is linked to the + `mri_scan_type`, `bids_category`, `bids_phase_encoding_direction`, `bids_scan_type` and `bids_scan_type_subcategory` - * the `bids_category*` table contains the different BIDS categories (or data type) linked along with the + * the `bids_category*` table contains the different BIDS categories (or data type) linked along with the `BIDSCategoryID` field used in `bids_mri_scan_type_rel` (examples: `anat`, `func`, `dwi`, `fmap`, `asl`...) * the `bids_scan_type*` table contains the different BIDS scan types (or modalities) linked along with the @@ -377,74 +377,74 @@ First, all DICOMs are converted into either MINC format (using `dcm2mnc` * the `bids_phase_encoding_direction*` table contains the different phase encoding directions of an image. It is linked along with the `BIDSPhaseEncodingDirectionID` field used in `bids_mri_scan_type_rel`. (examples: `i`, `-i`, `j`, `-j`, `k`, `-k`) - + `*` refers to tables used to determine BIDS file names when converting DICOM files into a BIDS dataset or previously inserted MINC files into a BIDS dataset. -Every converted file that matches the protocol defined in the tables mentioned above +Every converted file that matches the protocol defined in the tables mentioned above will be inserted in the database using the following tables: - - * the `files` table contains the information about the MINC file itself - (its location, the identified scan type, the file type...). Each row - correspond to one MINC file identified by the `SeriesUID` and `EchoTime` + + * the `files` table contains the information about the MINC file itself + (its location, the identified scan type, the file type...). Each row + correspond to one MINC file identified by the `SeriesUID` and `EchoTime` header information. - * the `parameter_file` table contains all the information stored in the - MINC header. Each row in that table stores a combination of a specific - header for a specific MINC file. This table is linked to the `files` - table using the foreign key `FileID` and to the `parameter_type` - table using the foreign key `ParameterTypeID`. Note: The `parameter_type` + * the `parameter_file` table contains all the information stored in the + MINC header. Each row in that table stores a combination of a specific + header for a specific MINC file. This table is linked to the `files` + table using the foreign key `FileID` and to the `parameter_type` + table using the foreign key `ParameterTypeID`. Note: The `parameter_type` table is automatically populated with the insertion of the first MINC - file in the database and stores the data dictionary for each MINC header + file in the database and stores the data dictionary for each MINC header field. * the `ImagingFileTypes` table contains the different file format that - can be inserted into the `files` table (`.mnc`, `.txt`, `.xml`...). The - field `type` of the table `ImagingFileTypes` is linked to the `FileType` + can be inserted into the `files` table (`.mnc`, `.txt`, `.xml`...). The + field `type` of the table `ImagingFileTypes` is linked to the `FileType` field of the `files` table. * the `mri_scanner` table contains information specific to the scanner - used to obtain the images. By convention, each scanner is assigned a - candidate in the `candidate` table which is linked to the `mri_scanner` - table using the `CandID` foreign key. In addition, the `ID` field of the - `mri_scanner` table is linked to the `ScannerID` field of the `files` + used to obtain the images. By convention, each scanner is assigned a + candidate in the `candidate` table which is linked to the `mri_scanner` + table using the `CandID` foreign key. In addition, the `ID` field of the + `mri_scanner` table is linked to the `ScannerID` field of the `files` table. - + ![files_tables](images/files_tables.png) Once an image has been inserted into the database, it is possible to view it - directly via the _Imaging Browser_ module under the _Imaging_ menu. + directly via the _Imaging Browser_ module under the _Imaging_ menu. #### 3.2.1.4 MRI violation tables -In the event a scan does not match any of the protocol mentioned in the +In the event a scan does not match any of the protocol mentioned in the `mri_protocol` table, LORIS automatically flags it as a violated scan. - Below is the description of the different tables involved in the + Below is the description of the different tables involved in the organization of such scans: - - * the `MRICandidateErrors` table stores scans for which the `PSCID` and + + * the `MRICandidateErrors` table stores scans for which the `PSCID` and the `CandID` fields stored in the `PatientName` of the DICOMs do not match any of the registered candidates in the `candidate` table. This is linked to the `tarchive` table via the `TarchiveID` foreign key. * the `mri_violations_log` table stores files for which a specific DICOM field - does not match the requirement specified in the `mri_protocol_checks` + does not match the requirement specified in the `mri_protocol_checks` * the `mri_protocol_violated_scans` table stores the violated scans' parameters (TR, TE, TI...) for easy identification of what is different - between the violated scan and the imaging protocol specified in the - `mri_protocol` table. This table is linked to the `tarchive` table via + between the violated scan and the imaging protocol specified in the + `mri_protocol` table. This table is linked to the `tarchive` table via the `TarchiveID` foreign key and to the `candidate` table via the `CandID` and `PSCID` foreign keys. - * the `violations_resolved` is linked to the three other tables mentioned - in this section. For each entry in that table, the `TypeTable` field + * the `violations_resolved` is linked to the three other tables mentioned + in this section. For each entry in that table, the `TypeTable` field allows to specify the table to link `violations_resolved` to and the `ExtID` allows to specify the ID to use from the linked table. Below is a table illustrating this concept. - + | TableType | ExtID | |--------------------------------|--------------------------------------| | MRICandidateErrors | MRICandidateErrors.ID | | mri\_violations\_log | mri\_violations\_log.LogID | -| mri\_protocol\_violated\_scans | mri\_protocol\_violated\_scans.ID | - +| mri\_protocol\_violated\_scans | mri\_protocol\_violated\_scans.ID | + ![violated_tables](images/violated_tables.png) #### 3.2.1.5 Quality Control (QC) tables @@ -452,15 +452,15 @@ In the event a scan does not match any of the protocol mentioned in the In the _Imaging Browser_ module, it is possible to view the images via _BrainBrowser_ and directly perform quality control of the images. The quality control information is then stored in the following tables: - - * the `files_qcstatus` table stores the QC status of the MINC file and - is linked to the `files` table via the `FileID` foreign key. + + * the `files_qcstatus` table stores the QC status of the MINC file and + is linked to the `files` table via the `FileID` foreign key. * the `feedback_mri_comments` table stores all the comments associated - to a given scan. These comments can be predefined (from the table - `feedback_mri_predefined_comments` or a text entered by the user + to a given scan. These comments can be predefined (from the table + `feedback_mri_predefined_comments` or a text entered by the user based on the comment type stored in `feedback_mri_comment_types`). - * session level QC information are saved in the `session` table and - session level comments are saved in the `feedback_mri_comments` + * session level QC information are saved in the `session` table and + session level comments are saved in the `feedback_mri_comments` table. ![qc_tables](images/QC_tables.png) @@ -472,34 +472,34 @@ Any native scan inserted into the `files` table can be processed and the output of this processing can be inserted into the database and linked to the native scan. For this, two additional tables require our attention (in light blue in the graphic below): - + * the `files_intermediary` table allows to link processed data with the - native datasets (or even intermediary outputs). The `Input_FileID` and + native datasets (or even intermediary outputs). The `Input_FileID` and `Output_FileID` fields of that table are links to the `FileID` field of the `files` table mentioned in section 3.2.3. Note that the native file - used to create processed outputs is always stored in the `files` table in + used to create processed outputs is always stored in the `files` table in the `SourceFileID` field, which is linked to the `FileID` field of the same table. - * the `mri_processing_protocol` table stores the imaging processing + * the `mri_processing_protocol` table stores the imaging processing protocols used to produce processed data. This table is linked to the `files` table using the `ProcessProtocolID` foreign key. Additionally, the field `FileType` of the `mri_processing_protocol` table is linked to the `type` field of the `ImagingFileTypes` table. - + ![processed_tables](images/Processed_data_tables.png) ### 3.2.2 Database infrastructure for the electrophysiology insertion pipeline -The database infrastructure for the electrophysiology data is composed of -five main tables: -- `physiological_file` and `physiological_parameter_file`: contain information +The database infrastructure for the electrophysiology data is composed of +five main tables: +- `physiological_file` and `physiological_parameter_file`: contain information specific to the electrophysiology file and a link to the `session` table -- `physiological_channel`: contains information regarding the channels used +- `physiological_channel`: contains information regarding the channels used to record the electrophysiology data if any provided in the BIDS structure -- `physiological_electrode`: contains information regarding the electrodes +- `physiological_electrode`: contains information regarding the electrodes used to record the electrophysiology data if any provided in the BIDS structure -- `physiological_task_event`: contains information about the task events that +- `physiological_task_event`: contains information about the task events that were used during the recording if any task was used in the paradigm ![OverallPhysiologicalTables](images/OverallPhysiologicalTables.png) @@ -507,17 +507,17 @@ were used during the recording if any task was used in the paradigm ### 3.2.2.1 The `physiological_file` and `physiological_parameter_file` tables -These two tables regroup information about the electrophysiology files. -Typically, the `physiologcal_file` table will store one recording per row linking it -to the `session` table. This table is also linked to the following tables +These two tables regroup information about the electrophysiology files. +Typically, the `physiologcal_file` table will store one recording per row linking it +to the `session` table. This table is also linked to the following tables using the `PhysiologicalFileID` key: -- `physiological_modality`: storing information about the recording modality +- `physiological_modality`: storing information about the recording modality (*e.g.* 'eeg', 'meg', 'ieeg'...) -- `physiological_output_type`: storing information about the BIDS output type +- `physiological_output_type`: storing information about the BIDS output type (*e.g.* 'raw' or 'derivatives') - `ImagingFileType`: storing the file type of the electrophysiology recording (*e.g.* 'set', 'bdf', 'cnt', 'edf', 'vhdr', 'vsm'...) -- `physiological_parameter_file`: containing all the information stored in the +- `physiological_parameter_file`: containing all the information stored in the BIDS JSON sidecar file of the electrophysiology recording ![PhysiologicalMainTables](images/PhysiologicalMainTables.png) @@ -525,27 +525,27 @@ BIDS JSON sidecar file of the electrophysiology recording ### 3.2.2.2 The `physiological_channel` table -The `physiological_channel` table is populated using data from the -`*_channels.tsv` BIDS file when present with the electrophysiology recording. -If the `*_channels.tsv` file is not present in the BIDS dataset, then no -entry will be inserted in this table. This table is linked to the -`physiological_file` table via the `PhysiologicalFileID` foreign key. In +The `physiological_channel` table is populated using data from the +`*_channels.tsv` BIDS file when present with the electrophysiology recording. +If the `*_channels.tsv` file is not present in the BIDS dataset, then no +entry will be inserted in this table. This table is linked to the +`physiological_file` table via the `PhysiologicalFileID` foreign key. In addition, `physiological_channel` is linked to the following tables: -- `physiological_status_type`: this table contains two entries specifying the +- `physiological_status_type`: this table contains two entries specifying the channel status ('good' and 'bad') -- `physiological_channel_type`: this contains information about the channel +- `physiological_channel_type`: this contains information about the channel type that was used during the recording (*e.g.* 'EEG', 'ECG', 'MEGMAG', 'ECOG' -...) along with their description +...) along with their description ![PhysiologicalChannelTables](images/PhysiologicalChannelTables.png) ### 3.2.2.3 The `physiological_electrode` table -The `physiological_electrode` table is populated using data from the -`*_electrodes.tsv` BIDS file when present with the electrophysiology recording. -If the `*_electrodes.tsv` file is not present in the BIDS dataset, then no -entry will be inserted in this table. This table is linked to the +The `physiological_electrode` table is populated using data from the +`*_electrodes.tsv` BIDS file when present with the electrophysiology recording. +If the `*_electrodes.tsv` file is not present in the BIDS dataset, then no +entry will be inserted in this table. This table is linked to the `physiological_file` table via the `PhysiologicalFileID` foreign key. In addition, `physiological_electrode` is linked to the following tables: - `physiological_electrode_type`: this table will store the different electrode @@ -559,10 +559,10 @@ table is unique. ### 3.2.2.4 The `physiological_task_event` table -The `physiological_task_event` table is populated using data from the -`*_events.tsv` BIDS file when present with the electrophysiology recording. -If the `*_events.tsv` file is not present in the BIDS dataset, then no -entry will be inserted in this table. This table is linked to the +The `physiological_task_event` table is populated using data from the +`*_events.tsv` BIDS file when present with the electrophysiology recording. +If the `*_events.tsv` file is not present in the BIDS dataset, then no +entry will be inserted in this table. This table is linked to the `physiological_file` table via the `PhysiologicalFileID` foreign key. ![PhysiologicalTaskTables](images/PhysiologicalTaskTables.png) @@ -570,10 +570,10 @@ entry will be inserted in this table. This table is linked to the ### 3.2.2.5 The `physiological_archive` table -The `physiological_archive` table will contain the path to the archive of all BIDS -files for a given physiological file (including the electrode.tsv, channel.tsv and +The `physiological_archive` table will contain the path to the archive of all BIDS +files for a given physiological file (including the electrode.tsv, channel.tsv and event.tsv files that comes with the electrophysiological recording). It was decided -to create an archive of those files for easier download directly from the View +to create an archive of those files for easier download directly from the View Session page of the Electrophysiological Browser module. This table is linked to the `physiological_file` table via the `PhysiologicalFileID` foreign key. diff --git a/docs/scripts_md/GetRole.md b/docs/scripts_md/GetRole.md index a58e13de0..a838c8b8a 100644 --- a/docs/scripts_md/GetRole.md +++ b/docs/scripts_md/GetRole.md @@ -7,11 +7,11 @@ NeuroDB::objectBroker::GetRole -- A role for basic SELECT operations on the data # DESCRIPTION This class provides methods used to accomplish basic `SELECT` operations on the database. -It allows the retrieval of records using a simple `WHERE` clause of the form +It allows the retrieval of records using a simple `WHERE` clause of the form `WHERE constraint_1 AND constraint_2 AND ... AND constraint_n`. -Subparts of the `WHERE` clause can only be combined with `AND` (`OR` is not supported). -Each subpart of the `WHERE` clause specifies that a field must be either equal to a given -value, not equal to a given value, NULL or NOT NULL. +Subparts of the `WHERE` clause can only be combined with `AND` (`OR` is not supported). +Each subpart of the `WHERE` clause specifies that a field must be either equal to a given +value, not equal to a given value, NULL or NOT NULL. Classes using this role must implement methods `getColumnNames()`, `getTableName()` and `db()`. @@ -20,7 +20,7 @@ Classes using this role must implement methods `getColumnNames()`, `getTableName ### get($isCount, $columnValuesRef) Fetches the records in the table whose name is returned by method `getTableName()` that satisfy -specific constraints. +specific constraints. Note that this method is private. @@ -31,21 +31,21 @@ INPUTS: should have in order to be part of the result set (key: column name, value: column value). -RETURNS: +RETURNS: - a reference to an array of hash references. Every hash contains the values for a given row returned by the method call: the key/value pairs contain - the name of a column (see `@MRI_SCAN_TYPE_FIELDS`) and the value it + the name of a column (see `@MRI_SCAN_TYPE_FIELDS`) and the value it holds, respectively. As an example, suppose array `$r` contains the result of a - given call to this function. One would fetch the `Scan_type` of the 2nd record - returned using `$r->[1]->{'Scan_type'}`. + given call to this function. One would fetch the `MriScanTypeName` of the 2nd record + returned using `$r->[1]->{'MriScanTypeName'}`. If the method is called with `$isCount` set to true, then it will return - a reference to an array containing a single hash reference, its unique key being + a reference to an array containing a single hash reference, its unique key being `'COUNT(*)'` with the associated value set to the selected count. ### addWhereEquals($columnValuesRef, $k, $whereRef, $valsRef) Gets the string representation of a constraint of the form `field=value` and adds it -to the current set of `WHERE` clauses. Note that constraints specifying that a field +to the current set of `WHERE` clauses. Note that constraints specifying that a field must be NULL will be properly translated to `field IS NULL` (not `field=NULL`). INPUTS: @@ -55,13 +55,13 @@ INPUTS: - Reference on the array of values each field in `$columnValuesRef` must be equal or not equal to. -RETURNS: +RETURNS: - Nothing (adds an element to `@$whereRef`). ### addWhereFunction($columnValuesRef, $k, $whereRef, $valsRef) Gets the string representation of a constraint that uses an SQL function or operator. -Currently, only the operator `NOT` (i.e. field `NOT` equal to a given value or `NOT NULL`) is +Currently, only the operator `NOT` (i.e. field `NOT` equal to a given value or `NOT NULL`) is supported. INPUTS: @@ -71,13 +71,13 @@ INPUTS: - Reference on the array of values each field in `$columnValuesRef` must be equal or not equal to. -RETURNS: +RETURNS: - Nothing. Updates arrays `@$whereRef` (and `@$valsRef` if necessary). ### addWhereNotEquals($columnValuesRef, $k, $whereRef, $valsRef) Gets the string representation of a constraint of the form `field != value` and adds it to -the current set of `WHERE` clauses. Note that constraints specifying that a field must not be NULL +the current set of `WHERE` clauses. Note that constraints specifying that a field must not be NULL will be properly translated to `field IS NOT NULL` (not `field!=NULL`). INPUTS: @@ -87,13 +87,13 @@ INPUTS: - Reference on the array of values each field in `$columnValuesRef` must be equal or not equal to. -RETURNS: +RETURNS: - Nothing. Updates arrays `@$whereRef` (and `@$valsRef` if necessary). ### get($columnValuesRef) Fetches the records in the table whose name is returned by method `getTableName()` that satisfy -specific constraints. +specific constraints. INPUTS: - reference to a hash array that contains the constraints on the column values that the records @@ -102,25 +102,25 @@ INPUTS: a single value or a reference to a hash (this hash describes a constraint involving an SQL function or operator other than '='). Examples of a valid set of constraints: - { + { Field1 => 'Value1', Field2 => { NOT => 3 }, Field3 => undef } - -RETURNS: + +RETURNS: - a reference to an array of hash references. Every hash contains the values for a given row returned by the method call: the key/value pairs contain - the name of a column (as listed by `getColumnNames()`) and the value it + the name of a column (as listed by `getColumnNames()`) and the value it holds, respectively. As an example, suppose array `$r` contains the result of a - given call to this function. One would fetch the `Scan_type` of the 2nd record - returned using `$r->[1]->{'Scan_type'}`. + given call to this function. One would fetch the `MriScanTypeName` of the 2nd record + returned using `$r->[1]->{'MriScanTypeName'}`. ### getCount($columnValuesRef) -Fetches the number of records in the table whose name is returned by method `getTableName()` -that satisfy specific constraints. +Fetches the number of records in the table whose name is returned by method `getTableName()` +that satisfy specific constraints. INPUTS: - reference to a hash array that contains the constraints on the column values that the records @@ -129,12 +129,12 @@ INPUTS: a single value or a reference to a hash (this hash describes a constraint involving an SQL function or operator other than '='). Examples of a valid set of constraints: - { + { Field1 => 'Value1', Field2 => { NOT => 3 }, Field3 => undef } - -RETURNS: + +RETURNS: - the number of records found. diff --git a/docs/scripts_md/MriScanTypeOB.md b/docs/scripts_md/MriScanTypeOB.md index d3651a1d1..1e9901795 100644 --- a/docs/scripts_md/MriScanTypeOB.md +++ b/docs/scripts_md/MriScanTypeOB.md @@ -77,12 +77,12 @@ INPUTS: RETURNS: a reference to an array of hash references. Every hash contains the values for a given row returned by the method call: the key/value pairs contain - the name of a column (see `@MRI_SCAN_TYPE_FIELDS`) and the value it + the name of a column (see `@MRI_SCAN_TYPE_FIELDS`) and the value it holds, respectively. As an example, suppose array `$r` contains the result of a - given call to this function. One would fetch the `Scan_type` of the 2nd record - returned using `$r-`\[1\]->{'Scan\_type'}>. + given call to this function. One would fetch the `MriScanTypeName` of the 2nd record + returned using `$r-`\[1\]->{'MriScanTypeName'}>. If the method is called with `$isCount` set to true, then it will return - a reference to an array containing a single hash reference, its unique key being + a reference to an array containing a single hash reference, its unique key being `'COUNT(*)'` with the associated value set to the selected count. # TO DO diff --git a/docs/scripts_md/minc_to_bids_converter.md b/docs/scripts_md/minc_to_bids_converter.md index af297dc1f..391b79019 100644 --- a/docs/scripts_md/minc_to_bids_converter.md +++ b/docs/scripts_md/minc_to_bids_converter.md @@ -73,30 +73,30 @@ RETURNS: { "1" => { - 'fileID' => 'FileID value', - 'file' => 'file path', - 'echoTime' => 'Echo Time of the file', - 'AcquisitionProtocolID' => 'Scan type ID', - 'candID' => 'Candidate CandID', - 'sessionID' => 'Session ID', - 'visitLabel' => 'Visit Label', - 'echoNumber' => 'Echo Number of the scan', - 'seriesNumber' => 'Series Number of the scan', - 'imageType' => 'Image Type', - 'lorisScanType' => 'LORIS Scan Type name' + 'fileID' => 'FileID value', + 'file' => 'file path', + 'echoTime' => 'Echo Time of the file', + 'MriScanTypeID' => 'Scan type ID', + 'candID' => 'Candidate CandID', + 'sessionID' => 'Session ID', + 'visitLabel' => 'Visit Label', + 'echoNumber' => 'Echo Number of the scan', + 'seriesNumber' => 'Series Number of the scan', + 'imageType' => 'Image Type', + 'lorisScanType' => 'LORIS Scan Type name' }, "2" => { - 'fileID' => 'FileID value', - 'file' => 'file path', - 'echoTime' => 'Echo Time of the file', - 'AcquisitionProtocolID' => 'Scan type ID', - 'candID' => 'Candidate CandID', - 'sessionID' => 'Session ID', - 'visitLabel' => 'Visit Label', - 'echoNumber' => 'Echo Number of the scan', - 'seriesNumber' => 'Series Number of the scan', - 'imageType' => 'Image Type', - 'lorisScanType' => 'LORIS Scan Type name' + 'fileID' => 'FileID value', + 'file' => 'file path', + 'echoTime' => 'Echo Time of the file', + 'MriScanTypeID' => 'Scan type ID', + 'candID' => 'Candidate CandID', + 'sessionID' => 'Session ID', + 'visitLabel' => 'Visit Label', + 'echoNumber' => 'Echo Number of the scan', + 'seriesNumber' => 'Series Number of the scan', + 'imageType' => 'Image Type', + 'lorisScanType' => 'LORIS Scan Type name' } ... } @@ -162,7 +162,7 @@ OUTPUT: 'BIDSCategoryName' => 'BIDS category to use for the NIfTI file, aka anat, func, fmap, dwi...', 'BIDSScanTypeSubCategory' => 'BIDS subcategory to use for the NIfTI file, aka task-rest, task-memory...', 'BIDSEchoNumber' => 'Echo Number associated with the NIfTI file', - 'Scan_type' => 'label of the LORIS Scan type from the mri_scan_type table' + 'ScanType' => 'label of the LORIS Scan type from the mri_scan_type table' } Note: BIDSEchoNumber and BIDSScanTypeSubCategory can be null for a given NIfTI file. @@ -371,14 +371,14 @@ OUTPUT: ### grep\_acquisitionProtocolID\_from\_BIDS\_scan\_type($db\_handle, $bids\_scan\_type) -Greps the AcquisitionProtocolID associated to a BIDS magnitude file in the database. +Greps the MriScanTypeID associated to a BIDS magnitude file in the database. INPUTS: - $db\_handle : database handle - $bids\_scan\_type: name of the BIDS scan type (for example: magnitude) OUTPUT: - - AcquisitionProtocolID associated to the BIDS scan type file in the database + - MriScanTypeID associated to the BIDS scan type file in the database ### create\_BIDS\_magnitude\_files($phasediff\_filename, $magnitude\_files\_hash) diff --git a/docs/scripts_md/register_processed_data.md b/docs/scripts_md/register_processed_data.md index 183fe7c9c..8f504b2d1 100644 --- a/docs/scripts_md/register_processed_data.md +++ b/docs/scripts_md/register_processed_data.md @@ -71,7 +71,7 @@ RETURNS: scanner ID ### getAcqProtID($scanType, $dbh) -This function returns the `AcquisitionProtocolID` of the file to register in +This function returns the `MriScanTypeID` of the file to register in the database based on `scanType` in the `mri_scan_type` table. INPUTS: diff --git a/python/lib/database_lib/files.py b/python/lib/database_lib/files.py index 4fab37e06..cd281a737 100644 --- a/python/lib/database_lib/files.py +++ b/python/lib/database_lib/files.py @@ -139,20 +139,20 @@ def update_files(self, file_id, fields, values): def select_distinct_acquisition_protocol_id_per_tarchive_source(self, tarchive_id): """ - Get a list of distinct scan types (a.k.a. `AcquisitionProtocolID`) inserted into the `files` + Get a list of distinct scan types (a.k.a. `MriScanTypeID`) inserted into the `files` table for a given DICOM archive (a.k.a. `TarchiveSource`). :param tarchive_id: `TarchiveID` to use as the `TarchiveSource` to restrict the SELECT statement on :type tarchive_id: int - :return: list of scan types found (`AcquisitionProtocolID`) + :return: list of scan types found (`MriScanTypeID`) :rtype: list """ - query = "SELECT DISTINCT AcquisitionProtocolID FROM files WHERE TarchiveSource = %s" + query = "SELECT DISTINCT MriScanTypeID FROM files WHERE TarchiveSource = %s" results = self.db.pselect(query=query, args=(tarchive_id,)) - acquisition_protocol_id_list = [v["AcquisitionProtocolID"] for v in results] + acquisition_protocol_id_list = [v["MriScanTypeID"] for v in results] return acquisition_protocol_id_list @@ -166,7 +166,7 @@ def get_file_ids_and_series_number_per_scan_type_and_tarchive_id(self, tarchive_ :param scan_type_id: ID of the scan type to restrict the query on :type scan_type_id: int - :return: list of `FileID` and `SeriesNumber` for a given `TarchiveID` and `AcquisitionProtocolID` + :return: list of `FileID` and `SeriesNumber` for a given `TarchiveID` and `MriScanTypeID` :rtype: list """ @@ -174,7 +174,7 @@ def get_file_ids_and_series_number_per_scan_type_and_tarchive_id(self, tarchive_ "FROM files " \ " JOIN parameter_file USING(FileID) " \ " JOIN parameter_type USING(ParameterTypeID) " \ - "WHERE TarchiveSource = %s AND AcquisitionProtocolID = %s AND Name = %s" + "WHERE TarchiveSource = %s AND MriScanTypeID = %s AND Name = %s" return self.db.pselect(query=query, args=(tarchive_id, scan_type_id, "series_number")) diff --git a/python/lib/database_lib/mri_protocol.py b/python/lib/database_lib/mri_protocol.py index 86fa29035..4d82cf0ec 100644 --- a/python/lib/database_lib/mri_protocol.py +++ b/python/lib/database_lib/mri_protocol.py @@ -112,15 +112,15 @@ def get_bids_info_for_scan_type_id(self, scan_type_id): bids_scan_type.BIDSScanType, bmstr.BIDSEchoNumber, bids_phase_encoding_direction.BIDSPhaseEncodingDirectionName, - mst.Scan_type + mst.MriScanTypeName AS ScanType FROM bids_mri_scan_type_rel bmstr - JOIN mri_scan_type mst ON mst.ID = bmstr.MRIScanTypeID + JOIN mri_scan_type mst ON mst.MriScanTypeID = bmstr.MRIScanTypeID JOIN bids_category USING (BIDSCategoryID) JOIN bids_scan_type USING (BIDSScanTypeID) LEFT JOIN bids_scan_type_subcategory USING (BIDSScanTypeSubCategoryID) LEFT JOIN bids_phase_encoding_direction USING (BIDSPhaseEncodingDirectionID) WHERE - mst.ID = %s + mst.MriScanTypeID = %s """ results = self.db.pselect(query=query, args=(scan_type_id,)) diff --git a/python/lib/database_lib/mri_protocol_checks.py b/python/lib/database_lib/mri_protocol_checks.py index 1bdd02aac..49683919a 100644 --- a/python/lib/database_lib/mri_protocol_checks.py +++ b/python/lib/database_lib/mri_protocol_checks.py @@ -56,7 +56,7 @@ def get_list_of_possible_protocols_based_on_session_info( query = "SELECT * FROM mri_protocol_checks" \ " JOIN mri_protocol_checks_group_target mpcgt USING (MriProtocolChecksGroupID)" \ - " WHERE Scan_type = %s " + " WHERE MriScanTypeID = %s " query += " AND (mpcgt.ProjectID IS NULL OR mpcgt.ProjectID = %s)" \ if project_id else " AND mpcgt.ProjectID IS NULL" diff --git a/python/lib/database_lib/mri_scan_type.py b/python/lib/database_lib/mri_scan_type.py index 64f4759b6..850053b0e 100644 --- a/python/lib/database_lib/mri_scan_type.py +++ b/python/lib/database_lib/mri_scan_type.py @@ -47,11 +47,11 @@ def get_scan_type_name_from_id(self, scan_type_id): """ results = self.db.pselect( - query='SELECT Scan_type FROM mri_scan_type WHERE ID = %s', + query='SELECT MriScanTypeName FROM mri_scan_type WHERE MriScanTypeID = %s', args=(scan_type_id,) ) - return results[0]['Scan_type'] if results else None + return results[0]['MriScanTypeName'] if results else None def get_scan_type_id_from_name(self, scan_type_name): """ @@ -65,8 +65,8 @@ def get_scan_type_id_from_name(self, scan_type_name): """ results = self.db.pselect( - query='SELECT ID FROM mri_scan_type WHERE Scan_type = %s', + query='SELECT MriScanTypeID FROM mri_scan_type WHERE MriScanTypeName = %s', args=(scan_type_name,) ) - return results[0]['ID'] if results else None + return results[0]['MriScanTypeID'] if results else None diff --git a/python/lib/dcm2bids_imaging_pipeline_lib/nifti_insertion_pipeline.py b/python/lib/dcm2bids_imaging_pipeline_lib/nifti_insertion_pipeline.py index 36e79ccf5..c5a647093 100644 --- a/python/lib/dcm2bids_imaging_pipeline_lib/nifti_insertion_pipeline.py +++ b/python/lib/dcm2bids_imaging_pipeline_lib/nifti_insertion_pipeline.py @@ -637,7 +637,7 @@ def _register_violations_log(self, violations_list, file_rel_path): 'PatientName': self.subject_id_dict['PatientName'], 'CandID': self.subject_id_dict['CandID'], 'Visit_label': self.subject_id_dict['visitLabel'], - 'Scan_type': self.scan_type_id, + 'MriScanTypeID': self.scan_type_id, 'EchoTime': scan_param['EchoTime'] if 'EchoTime' in scan_param.keys() else None, 'EchoNumber': scan_param['EchoNumber'] if 'EchoNumber' in scan_param.keys() else None, 'PhaseEncodingDirection': phase_enc_dir, @@ -679,7 +679,7 @@ def _register_into_files_and_parameter_file(self, nifti_rel_path): 'PhaseEncodingDirection': phase_enc_dir, 'CoordinateSpace': 'native', 'OutputType': 'native', - 'AcquisitionProtocolID': self.scan_type_id, + 'MriScanTypeID': self.scan_type_id, 'FileType': file_type, 'InsertedByUserID': getpass.getuser(), 'InsertTime': datetime.datetime.now().timestamp(), diff --git a/python/lib/imaging.py b/python/lib/imaging.py index 3a9c7342b..832c0eafa 100644 --- a/python/lib/imaging.py +++ b/python/lib/imaging.py @@ -305,7 +305,7 @@ def insert_mri_violations_log(self, info_to_insert_dict): echo_number = repr(info_to_insert_dict["EchoNumber"]) phase_encoding_dir = info_to_insert_dict["PhaseEncodingDirection"] echo_time = info_to_insert_dict['EchoTime'] - scan_type = info_to_insert_dict['Scan_type'] + scan_type = info_to_insert_dict['MriScanTypeID'] severity = info_to_insert_dict['Severity'] header = info_to_insert_dict['Header'] value = info_to_insert_dict['Value'] @@ -320,7 +320,7 @@ def insert_mri_violations_log(self, info_to_insert_dict): if str(row['SeriesUID']) == str(series_uid) \ and str(row['PhaseEncodingDirection']) == str(phase_encoding_dir) \ and str(row['EchoNumber']) == str(echo_number) \ - and str(row['Scan_type']) == str(scan_type) \ + and str(row['MriScanTypeID']) == str(scan_type) \ and str(row['EchoTime']) == str(echo_time) \ and str(row['Severity']) == str(severity) \ and str(row['Header']) == str(header) \ @@ -673,15 +673,15 @@ def look_for_matching_protocols(self, protocols_list, scan_param, scan_type=None matching_protocols_list = [] for protocol in protocols_list: - if scan_type_id and protocol['Scan_type'] == scan_type_id: - matching_protocols_list.append(protocol['Scan_type']) + if scan_type_id and protocol['MriScanTypeID'] == scan_type_id: + matching_protocols_list.append(protocol['MriScanTypeID']) elif protocol['series_description_regex']: if re.search( rf"{protocol['series_description_regex']}", scan_param['SeriesDescription'], re.IGNORECASE ): - matching_protocols_list.append(protocol['Scan_type']) + matching_protocols_list.append(protocol['MriScanTypeID']) elif self.is_scan_protocol_matching_db_protocol(protocol, scan_param): - matching_protocols_list.append(protocol['Scan_type']) + matching_protocols_list.append(protocol['MriScanTypeID']) return list(dict.fromkeys(matching_protocols_list)) @@ -969,7 +969,7 @@ def get_list_of_fmap_files_sorted_by_acq_time(self, files_list): for file_dict in files_list: bids_info = self.mri_prot_db_obj.get_bids_info_for_scan_type_id( - file_dict['AcquisitionProtocolID'] + file_dict['MriScanTypeID'] ) param_file_result = self.param_file_db_obj.get_parameter_file_for_file_id_param_type_id( file_dict['FileID'], @@ -1030,7 +1030,7 @@ def get_list_of_files_sorted_by_acq_time(self, files_list): new_files_list = [] for file_dict in files_list: bids_info = self.mri_prot_db_obj.get_bids_info_for_scan_type_id( - file_dict['AcquisitionProtocolID'] + file_dict['MriScanTypeID'] ) param_file_result = self.param_file_db_obj.get_parameter_file_for_file_id_param_type_id( file_dict['FileID'], diff --git a/python/lib/mri.py b/python/lib/mri.py index 21335a956..391465366 100644 --- a/python/lib/mri.py +++ b/python/lib/mri.py @@ -366,9 +366,9 @@ def fetch_and_insert_nifti_file(self, nifti_file, derivatives=None): # grep the scan type ID from the mri_scan_type table (if it is not already in # the table, it will add a row to the mri_scan_type table) scan_type_id = self.db.grep_id_from_lookup_table( - id_field_name = 'ID', + id_field_name = 'MriScanTypeID', table_name = 'mri_scan_type', - where_field_name = 'Scan_type', + where_field_name = 'MriScanTypeName', where_value = scan_type, insert_if_not_found = True ) @@ -392,7 +392,7 @@ def fetch_and_insert_nifti_file(self, nifti_file, derivatives=None): 'PhaseEncodingDirection': phase_enc_dir, 'EchoNumber' : echo_nb, 'SourceFileID' : None, - 'AcquisitionProtocolID': scan_type_id + 'MriScanTypeID' : scan_type_id } file_id = imaging.insert_imaging_file(file_info, file_parameters) diff --git a/tools/delete_imaging_upload.pl b/tools/delete_imaging_upload.pl index a3c6aa936..386c19086 100755 --- a/tools/delete_imaging_upload.pl +++ b/tools/delete_imaging_upload.pl @@ -6,7 +6,7 @@ =head1 NAME delete_imaging_upload.pl -- Delete everything that was produced (or part of what was produced) by the imaging pipeline for a given set of imaging uploads IDs, all associated to the same C. - + =head1 SYNOPSIS perl delete_imaging_upload.pl [-profile file] [-ignore] [-backup_path basename] [-protocol] [-form] [-uploadID list_of_uploadIDs] @@ -18,49 +18,49 @@ =head1 SYNOPSIS -ignore : ignore files whose paths exist in the database but do not exist on the file system. Default is to abort if such a file is found, irrespective of whether a backup file will - be created by the script (see C<-nofilesbk> and C<-nosqlbk>). If this option is used, a + be created by the script (see C<-nofilesbk> and C<-nosqlbk>). If this option is used, a warning is issued and program execution continues. - + -nofilesbk : when creating the backup file for the deleted upload(s), do not backup the files produced by the imaging pipeline (default is to backup these files). - + -backup_path : specify the path of the backup file, which by default contains a copy of everything that the script will delete, both on the file system and in the database (but see C<-nofilesbk> and C<-nosqlbk>). The extension C<.tar.gz> will be added to this base name to build the name of the final backup file. If a file with this resulting name already exists, an error message is shown and the script - will abort. Note that C can be an absolute path or a path relative to the current directory. A + will abort. Note that C can be an absolute path or a path relative to the current directory. A backup file is always created unless options C<-nofilesbk> and C<-nosqlbk> are both used. By default, the - backup file name is C and is written in the current directory. Option + backup file name is C and is written in the current directory. Option C<-backup_path> cannot be used if C<-nofilesbk> and C<-nosqlbk> are also used. - --basename fileBaseName : basename of the file to delete. The file is assumed to either exist in table C or table + +-basename fileBaseName : basename of the file to delete. The file is assumed to either exist in table C or table C. This option should be used when targeting a specific (unique) file for deletion. - Note that the file will be deleted from both the database and the filesystem. This option cannot be + Note that the file will be deleted from both the database and the filesystem. This option cannot be used with options C<-defaced> and C<-form>. - --uploadID : comma-separated list of upload IDs (found in table C) to delete. The program will + +-uploadID : comma-separated list of upload IDs (found in table C) to delete. The program will abort if the list contains an upload ID that does not exist. Also, all upload IDs must have the same C ID (which can be C). - + -protocol : delete the imaging protocol(s) in table C associated to either the upload(s) specified via the C<-uploadID> option or any file that was produced using this (these) upload(s). Let F be the set of files directly or indirectly associated to the upload(s) to delete. This option must be used if there is at least one record in C that is tied only to files in F. Protocols that are tied to files not in F are never deleted. If the files in F do not have a protocol associated to them, the switch is ignored if used. - + -form : delete the entries in C associated to the upload(s) passed on the command line, if any (default is NOT to delete them). - + -type : comma-separated list of scan type names to delete. All the names must exist in table C or the script will issue an error. This option cannot be used in conjunction with C<-defaced>. - + -defaced : fetch the scan types listed in config setting C and perform a deletion of these scan types as if their names were used with option C<-type>. Once all deletions are done, set the C - and C of all the defaced files in table to C and to the tarchive ID of the + and C of all the defaced files in table to C and to the tarchive ID of the upload(s) whose arguments were passed to C<-uploadID>, respectively. - --nosqlbk : when creating the backup file, do not add to it an SQL file that contains the statements used to restore + +-nosqlbk : when creating the backup file, do not add to it an SQL file that contains the statements used to restore the database to the state it had before the script was invoked. Adding this file, which will be named C, to the backup file is the default behaviour. @@ -89,7 +89,7 @@ =head1 DESCRIPTION b) C c) C and C d) C, C and C - e) C. + e) C. f) C g) C h) C @@ -99,29 +99,29 @@ =head1 DESCRIPTION All the deletions and modifications performed in the database are done as part of a single transaction, so they either all succeed or a rollback is performed and the database is not modified in any way. The ID of the upload to delete -is specified via option C<-uploadID>. More than one upload can be deleted if they all have the same C +is specified via option C<-uploadID>. More than one upload can be deleted if they all have the same C in table C: option C<-uploadID> can take as argument a comma-separated list of upload IDs for this case. If an upload that is deleted is the only one that was associated to a given session, the script will set the C -value for that session to 'N'. If option C<-form> is used, the C and its associated C record -are also deleted, for each deleted upload. If option C<-protocol> is used and if there is a record in table +value for that session to 'N'. If option C<-form> is used, the C and its associated C record +are also deleted, for each deleted upload. If option C<-protocol> is used and if there is a record in table C that is tied only to the deleted upload(s), then that record is also deleted. C cannot be used to delete an upload that has an associated MINC file that has been QCed. -In other words, if there is a MINC file tied to the upload that is targeted for deletion and if that MINC file has +In other words, if there is a MINC file tied to the upload that is targeted for deletion and if that MINC file has an associated record in table C or C, the script will issue an error message and exit. -Before deleting any records in the database, the script will verify that all the records in tables a) through j) that +Before deleting any records in the database, the script will verify that all the records in tables a) through j) that represent file names (e.g. a record in C, a protocol file in C, etc...) refer to files -that actually exist on the file system. If it finds a record that does not meet that criterion, the script issues an +that actually exist on the file system. If it finds a record that does not meet that criterion, the script issues an error message and exits, leaving the database untouched. To avoid this check, use option C<-ignore>. Each time a file -record is deleted, the file it refers to on the file system is also deleted. A backup will be created by -C of all the files that were deleted during execution. Option C<-nofilesbk> can be used to +record is deleted, the file it refers to on the file system is also deleted. A backup will be created by +C of all the files that were deleted during execution. Option C<-nofilesbk> can be used to prevent this. If created, the backup file will be named C. This name can be changed with option C<-backup_path>. Note that the file paths inside this backup archive are absolute. To restore the files in the archive, one must use C with option C<--absolute-names>. -The script will also create a file that contains a backup of all the information that was deleted or modified from the +The script will also create a file that contains a backup of all the information that was deleted or modified from the database tables. This backup is created using C and contains an C statement for every record erased. When running C the script uses the database credentials in file C<~/.my.cnf> to connect to the database. Option C<-extra_mysqlcnf> has to be used to specify an alternate credentials file when the default credential file does not exist @@ -131,33 +131,33 @@ =head1 DESCRIPTION C was invoked, provided the database was not modified in the meantime. The SQL backup file will be named C. -2. Delete specific scan types from an archive. The behaviour of the script is identical to the one described above, except +2. Delete specific scan types from an archive. The behaviour of the script is identical to the one described above, except that: a) the deletions are limited to MINC files of a specific scan type: use option C<-type> with a comma-separated list of scan type names to specify which ones. - b) everything associated to the MINC files deleted in a) is also deleted: this includes the processed files in + b) everything associated to the MINC files deleted in a) is also deleted: this includes the processed files in C and the records in C. c) if C<-protocol> is used and there is an entry in table C that is tied only to the files deleted in a), then that record is also deleted. d) tables C, C, C, C, C, C and C are never modified. Note that option C<-type> cannot be used in conjunction with either option C<-form> or option C<-defaced>. - -3. Replace MINC files with their defaced counterparts. This is the behaviour obtained when option C<-defaced> is used. As far as - deletions go, the behaviour of the script in this case is identical to the one described in 2), except that the list of + +3. Replace MINC files with their defaced counterparts. This is the behaviour obtained when option C<-defaced> is used. As far as + deletions go, the behaviour of the script in this case is identical to the one described in 2), except that the list of scan types to delete is fetched from the config setting C. Use of option C<-defaced> is not permitted - in conjunction with option C<-type> or option C<-form>. Once all deletions are made, the script will change the C - of all defaced files to C and set the C of all defaced files to the C of the upload(s). - This effectively "replaces" the original MINC files with their corresponding defaced versions. Note that the script will issue + in conjunction with option C<-type> or option C<-form>. Once all deletions are made, the script will change the C + of all defaced files to C and set the C of all defaced files to the C of the upload(s). + This effectively "replaces" the original MINC files with their corresponding defaced versions. Note that the script will issue an error message and abort, leaving the database and file system untouched, if: a) A MINC file should have a corresponding defaced file but does not. b) A MINC file that has been defaced has an associated file that is not a defaced file. 4. Delete a specific file that exists in table C or table C (in which case it will be associated to parameter C, C, C or C). Note that once a file is - deleted, all the files that have been derived/built using this deleted file will also be deleted. Use option C<-basename fileBaseName>, + deleted, all the files that have been derived/built using this deleted file will also be deleted. Use option C<-basename fileBaseName>, where C is the basename of the file to delete, along with option C<-uploadID> to use the script this way. - + =head2 Methods @@ -192,7 +192,7 @@ =head2 Methods use constant DEFAULT_DELETE_MRI_PARAMETER_FORM => 0; use constant DEFAULT_KEEP_DEFACED => 0; -use constant FILE_PARAMETER_NAMES => ('check_pic_filename', 'check_nii_filename', 'check_bval_filename', 'check_bvec_filename'); +use constant FILE_PARAMETER_NAMES => ('check_pic_filename', 'check_nii_filename', 'check_bval_filename', 'check_bvec_filename'); # Name that the SQL file will have inside the tar.gz backup archive use constant SQL_RESTORE_NAME => 'imaging_upload_restore.sql'; @@ -217,7 +217,7 @@ =head2 Methods ); # Stolen from get_dicom_files.pl: should be a constant global to both -# scripts +# scripts my $FLOAT_EQUALS_THRESHOLD = 0.00001; my %options = ( @@ -238,11 +238,11 @@ =head2 Methods my $scanTypeList = undef; my @opt_table = ( - ['-profile' , 'string' , 1, \$options{'PROFILE'}, + ['-profile' , 'string' , 1, \$options{'PROFILE'}, 'Name of config file in ../dicom-archive/.loris_mri (defaults to "prod")'], - ['-backup_path', 'string' , 1, \$options{'BACKUP_PATH'}, + ['-backup_path', 'string' , 1, \$options{'BACKUP_PATH'}, 'Path of the backup file (defaults to "imaging_upload_backup", in the current directory)'], - ['-ignore' , 'const' , 0, \$options{'DIE_ON_FILE_ERROR'}, + ['-ignore' , 'const' , 0, \$options{'DIE_ON_FILE_ERROR'}, 'Ignore files that exist in the database but not on the file system.' . ' Default is to abort if such a file is found.'], ['-nofilesbk' , 'const' , 1, \$options{'NO_FILES_BK'}, @@ -292,32 +292,32 @@ =head2 Methods if(defined $scanTypeList && $options{'DELETE_MRI_PARAMETER_FORM'}) { print STDERR "Option -type cannot be used in conjunction with option -form. Aborting.\n"; exit $NeuroDB::ExitCodes::INVALID_ARG; -} +} if($options{'BASENAME'} =~ /\// ) { print STDERR "Argument $options{'BASENAME'} to -basename is not a file basename (contains a '/'). Aborting.\n"; exit $NeuroDB::ExitCodes::INVALID_ARG; -} +} if($options{'BASENAME'} ne '' && $options{'DELETE_MRI_PARAMETER_FORM'}) { print STDERR "Option -basename cannot be used in conjunction with option -form. Aborting.\n"; exit $NeuroDB::ExitCodes::INVALID_ARG; -} +} if(defined $scanTypeList && $options{'KEEP_DEFACED'}) { print STDERR "Option -type cannot be used in conjunction with option -defaced. Aborting.\n"; exit $NeuroDB::ExitCodes::INVALID_ARG; -} +} if($options{'KEEP_DEFACED'} && $options{'DELETE_MRI_PARAMETER_FORM'}) { print STDERR "Option -defaced cannot be used in conjunction with option -form. Aborting.\n"; exit $NeuroDB::ExitCodes::INVALID_ARG; -} +} if($options{'KEEP_DEFACED'} && $options{'BASENAME'} ne '') { print STDERR "Option -defaced cannot be used in conjunction with option -basename. Aborting.\n"; exit $NeuroDB::ExitCodes::INVALID_ARG; -} +} if($options{'UPLOAD_ID'} eq '') { print STDERR "Missing -uploadID option. Aborting.\n"; @@ -360,12 +360,12 @@ =head2 Methods if (!$ENV{LORIS_CONFIG}) { print STDERR "\n\tERROR: Environment variable 'LORIS_CONFIG' not set\n\n"; - exit $NeuroDB::ExitCodes::INVALID_ENVIRONMENT_VAR; + exit $NeuroDB::ExitCodes::INVALID_ENVIRONMENT_VAR; } if (!-e "$ENV{LORIS_CONFIG}/.loris_mri/$options{'PROFILE'}") { - print $Help; - print STDERR "Cannot read profile file '$ENV{LORIS_CONFIG}/.loris_mri/$options{'PROFILE'}'\n"; + print $Help; + print STDERR "Cannot read profile file '$ENV{LORIS_CONFIG}/.loris_mri/$options{'PROFILE'}'\n"; exit $NeuroDB::ExitCodes::PROFILE_FAILURE; } @@ -409,8 +409,8 @@ =head2 Methods # been commited at that point will automatically be rolled back $dbh->{'RaiseError'} = 1; -# Do not print an error message when a DB error occurs since the error message -# will be printed when the program dies anyway +# Do not print an error message when a DB error occurs since the error message +# will be printed when the program dies anyway $dbh->{'PrintError'} = 0; my %files; @@ -437,7 +437,7 @@ =head2 Methods die "Invalid scan types argument: " . join(',', @invalidTypes) . "\n" if @invalidTypes; my @scanTypesToDelete = keys %scanTypesToDelete; die "Option -form cannot be used in conjunction with -type. Aborting\n" if $options{'DELETE_MRI_PARAMETER_FORM'} && @scanTypesToDelete; - + #=================================================================# # Find the absolute paths of all files associated to the # # upload(s) passed on the command lines in all the tables listed # @@ -451,7 +451,7 @@ =head2 Methods printf STDERR "option %s cannot be used. Aborting.\n", ($options{'KEEP_DEFACED'} ? '-defaced' : '-type'); exit $NeuroDB::ExitCodes::INVALID_ARG; } - if($options{'BASENAME'} ne '') { + if($options{'BASENAME'} ne '') { print STDERR "The upload(s) specified on the command line are not tied to any records in table tarchive: "; printf STDERR "option -basename cannot be used. Aborting.\n"; exit $NeuroDB::ExitCodes::INVALID_ARG; @@ -472,7 +472,7 @@ =head2 Methods print STDERR "Cannot determine absolute path for archive $tarchiveID since config " . "setting 'tarchiveLibraryDir' is not set. Aborting.\n"; exit $NeuroDB::ExitCodes::MISSING_CONFIG_SETTING; -} +} #=================================================================# # If -defaced was used, validate that: # # a) All the MINCs that should have been defaced were defaced # @@ -491,7 +491,7 @@ =head2 Methods printf STDERR join("", map { "\t$_\n" } @$invalidFilesRef); } } - print STDERR "\nAborting.\n"; + print STDERR "\nAborting.\n"; exit $NeuroDB::ExitCodes::INVALID_ARG; } } @@ -533,7 +533,7 @@ =head2 Methods $msg .= "\t$_\n"; } } - + die "Error! $msg\nAborting.\n" if $options{'DIE_ON_FILE_ERROR'}; print STDERR "Warning! $msg\n"; } @@ -575,14 +575,14 @@ =head2 Methods =head3 printExitMessage($filesRef, $scanTypesToDeleteRef, $deleteResultsRef) -Prints an appropriate message before exiting. +Prints an appropriate message before exiting. INPUTS: - $filesRef: reference to the array that contains the file information for all the files that are associated to the upload(s) passed on the command line. - $scanTypesToDeleteRef: reference to the array that contains the list of scan type names to delete. - $deleteResultsRef: reference on the hash that contains the result of the deletion of the records in the - database. + database. =cut sub printExitMessage { @@ -593,7 +593,7 @@ sub printExitMessage { if(@$scanTypesToDeleteRef) { if(!$deleteResultsRef->{'NB_RECORDS_DELETED'}) { printf( - "No scans of type %s found for upload %s.\n", + "No scans of type %s found for upload %s.\n", &prettyListPrint($scanTypesToDeleteRef, 'or'), $filesRef->{'mri_upload'}->[0]->{'UploadID'} ); @@ -616,7 +616,7 @@ sub printExitMessage { =pod -=head3 prettyListPrint($listRef, $andOr) +=head3 prettyListPrint($listRef, $andOr) Pretty prints a list in string form (e.g "1, 2, 3 and 4" or "7, 8 or 9"). @@ -627,9 +627,9 @@ =head3 prettyListPrint($listRef, $andOr) =cut sub prettyListPrint { my($listRef, $andOr) = @_; - + return $listRef->[0] if @$listRef == 1; - + return join(', ', @$listRef[0..$#{ $listRef }-1]) . " $andOr " . $listRef->[$#{ $listRef } ]; } @@ -651,7 +651,7 @@ =head3 getMriUploadFiles($dbh, $uploadIDsRef) sub getMriUploadFiles { my($dbh, $uploadIDsRef) = @_; - + my $query = "SELECT UploadID, TarchiveID, UploadLocation as FullPath, Inserting, InsertionComplete, SessionID " . "FROM mri_upload " . "WHERE UploadID IN (" @@ -678,25 +678,25 @@ =head3 getTarchiveFiles($dbh, $tarchiveID, $tarchiveLibraryDir) - a reference on an array that contains the C associated to the upload ID(s) passed on the command line. This array can contain one element (if the uploads are all tied to the same C) or be empty (if all the uploads have a C set to C). If the C for the C is a relative path and - if config setting Cis not defined, the return value is C, indicating that something is + if config setting Cis not defined, the return value is C, indicating that something is wrong. =cut sub getTarchiveFiles { my($dbh, $tarchiveID, $tarchiveLibraryDir) = @_; - + return [] if !defined $tarchiveID; - + my $query = "SELECT TarchiveID, ArchiveLocation " . "FROM tarchive " . "WHERE TarchiveID = ?"; my $tarchiveRef = $dbh->selectall_arrayref($query, { Slice => {} }, $tarchiveID); my $archiveLocation = $tarchiveRef->[0]->{'ArchiveLocation'}; - + return undef if defined $archiveLocation && $archiveLocation !~ /^\// && !defined $tarchiveLibraryDir; - - $tarchiveRef->[0]->{'FullPath'} = !defined $archiveLocation || $archiveLocation =~ /^\// + + $tarchiveRef->[0]->{'FullPath'} = !defined $archiveLocation || $archiveLocation =~ /^\// ? $archiveLocation : "$tarchiveLibraryDir/$archiveLocation", return $tarchiveRef; @@ -720,7 +720,7 @@ =head3 validateMriUploads($mriUploadsRef, $optionsRef, $scanTypeList) sub validateMriUploads { my($mriUploadsRef, $optionsRef, $scanTypeList) = @_; - + #======================================================# # Check that all upload IDs passed on the command line # # were found in the database # @@ -742,7 +742,7 @@ sub validateMriUploads { exit $NeuroDB::ExitCodes::INVALID_ARG; } } - + #===============================================================================# # Get the tarchive IDs of all the uploads. Note that if an upload does not # # have a tarchiveID, its value will be undef: we remap it to the string 'NULL' # @@ -761,7 +761,7 @@ sub validateMriUploads { # If the upload IDs to delete are associated to an exsisting archive, ensure # # that there are no other uploads with IDs *not* in the list associated to that # # archive. This check is done only if the user wants to delete the uploads and # - # everything associated to them. + # everything associated to them. #===============================================================================# my @tarchiveID = keys %tarchiveID; if($tarchiveID[0] ne 'NULL') { @@ -789,7 +789,7 @@ sub validateMriUploads { =head3 getMriProcessingProtocolFilesRef($dbh, $filesRef) Finds the list of Cs to delete, namely those in table -C associated to the files to delete, and *only* to +C associated to the files to delete, and *only* to those files that are going to be deleted. INPUTS: @@ -799,37 +799,37 @@ =head3 getMriProcessingProtocolFilesRef($dbh, $filesRef) RETURNS: - reference on an array that contains the C in table C - associated to the files to delete. This array has two keys: C => the protocol + associated to the files to delete. This array has two keys: C => the protocol process ID found in table C and C => the value of C in the same table. =cut sub getMriProcessingProtocolFilesRef { my($dbh, $filesRef) = @_; - + my %fileID; foreach my $t (@PROCESSED_TABLES) { foreach my $f (@{ $filesRef->{$t} }) { $fileID{ $f->{'FileID'} } = 1 if defined $f->{'FileID'}; } } - + return [] if !%fileID; - + my $query = 'SELECT DISTINCT(mpp.ProcessProtocolID), mpp.ProtocolFile AS FullPath ' . 'FROM files f ' . 'JOIN mri_processing_protocol mpp USING (ProcessProtocolID) ' . 'WHERE f.FileID IN (' . join(',', ('?') x keys %fileID) . ') AND f.ProcessProtocolID IS NOT NULL'; - + # This variable will contain all the protocols associated to the files that will be - # deleted. Note that these protocols might also be associated to files that are *not* + # deleted. Note that these protocols might also be associated to files that are *not* # going to be deleted my $protocolsRef = $dbh->selectall_arrayref($query, { Slice => {} }, keys %fileID); - + return [] unless @$protocolsRef; - + # Now get the list of protocols in @$protocolsRef that are also associated to files # that are not going to de deleted $query = 'SELECT ProcessProtocolID ' @@ -840,21 +840,21 @@ sub getMriProcessingProtocolFilesRef { . 'AND FileID NOT IN (' . join(',', ('?') x keys %fileID) . ')'; - + my $protocolsNotDeletedRef = $dbh->selectcol_arrayref( - $query, + $query, { 'Columns' => [1] }, - (map { $_->{'ProcessProtocolID'} } @$protocolsRef), + (map { $_->{'ProcessProtocolID'} } @$protocolsRef), keys %fileID ); - + # Eliminate from @$protocolsRef the entries of that are also in @$protocolsNotDeletedRef - for(my $i=$#{ $protocolsRef }; $i >= 0; $i--) { + for(my $i=$#{ $protocolsRef }; $i >= 0; $i--) { if(grep($_ == $protocolsRef->[$i]->{'ProcessProtocolID'}, @$protocolsNotDeletedRef)) { splice(@$protocolsRef, $i, 1); } } - + return $protocolsRef; } @@ -862,8 +862,8 @@ sub getMriProcessingProtocolFilesRef { =head3 hasQcOrComment($dbh, $mriUploadsRef) -Determines if any of the MINC files associated to the C have QC -information associated to them by looking at the contents of tables +Determines if any of the MINC files associated to the C have QC +information associated to them by looking at the contents of tables C and C. INPUTS: @@ -879,25 +879,25 @@ =head3 hasQcOrComment($dbh, $mriUploadsRef) =cut sub hasQcOrComment { my($dbh, $mriUploadsRef) = @_; - + # Note: all uploads have the same TarchievID so examining the first one is OK my $tarchiveID = $mriUploadsRef->[0]->{'TarchiveID'}; - return 0 if !defined $tarchiveID; - + return 0 if !defined $tarchiveID; + #=========================================# # Fetch contents of tables files_qcstatus # # and feedback_mri_comments # #=========================================# (my $query =<. INPUTS: @@ -922,7 +922,7 @@ =head3 getFilesRef($dbh, $tarchiveID, $dataDirBasePath, $scanTypesToDeleteRef, $ - $scanTypesToDeleteRef: reference to the array that contains the list of names of scan types to delete. - $optionsRef: reference on the array of command line options. -RETURNS: +RETURNS: - an array of hash references. Each hash has three keys: C => ID of a file in table C, C => value of column C for the file with the given ID and C => absolute path for the file with the given ID. @@ -930,47 +930,47 @@ =head3 getFilesRef($dbh, $tarchiveID, $dataDirBasePath, $scanTypesToDeleteRef, $ =cut sub getFilesRef { my($dbh, $tarchiveID, $dataDirBasePath, $scanTypesToDeleteRef, $optionsRef) = @_; - + return [] if !defined $tarchiveID; - + my $fileBaseName = $optionsRef->{'BASENAME'}; # Get FileID and File path of each files in files directly tied # to $tarchiveId - my $mriScanTypeJoin = @$scanTypesToDeleteRef - ? 'JOIN mri_scan_type mst ON (f.AcquisitionProtocolID = mst.ID) ' + my $mriScanTypeJoin = @$scanTypesToDeleteRef + ? 'JOIN mri_scan_type mst ON (f.MriScanTypeID = mst.MriScanTypeID) ' : ''; - my $mriScanTypeAnd = @$scanTypesToDeleteRef - ? sprintf(' AND mst.Scan_type IN (%s) ', join(',', ('?') x @$scanTypesToDeleteRef)) + my $mriScanTypeAnd = @$scanTypesToDeleteRef + ? sprintf(' AND mst.MriScanTypeName IN (%s) ', join(',', ('?') x @$scanTypesToDeleteRef)) : ''; my $fileBaseNameAnd = $fileBaseName ne '' ? ' AND BINARY SUBSTRING_INDEX(f.File, "/", -1) = ?' : ''; - my $query = 'SELECT f.FileID, f.File FROM files f ' + my $query = 'SELECT f.FileID, f.File FROM files f ' . $mriScanTypeJoin . 'WHERE f.TarchiveSource = ? ' . $mriScanTypeAnd . $fileBaseNameAnd; - + my @queryArgs = ($tarchiveID, @$scanTypesToDeleteRef); push(@queryArgs, $fileBaseName) if $fileBaseName ne ''; my $filesRef = $dbh->selectall_arrayref($query, { Slice => {} }, @queryArgs); - + # Set full path of every file foreach(@$filesRef) { - $_->{'FullPath'} = $_->{'File'} =~ /^\// + $_->{'FullPath'} = $_->{'File'} =~ /^\// ? $_->{'File'} : "$dataDirBasePath/$_->{'File'}"; - } - - return $filesRef; + } + + return $filesRef; } =pod =head3 getIntermediaryFilesRef($dbh, $filesRef, $tarchiveID, $dataDirBasePath, $scanTypesToDeleteRef, $optionsRef) -Get the absolute paths of all the intermediary files associated to an archive +Get the absolute paths of all the intermediary files associated to an archive that are listed in table C. INPUTS: @@ -981,18 +981,18 @@ =head3 getIntermediaryFilesRef($dbh, $filesRef, $tarchiveID, $dataDirBasePath, $ - $scanTypesToDeleteRef: reference to the array that contains the list of scan type names to delete. - $optionsRef: reference on the array of command line options. -RETURNS: - - an array of hash references. Each hash has seven keys: C => ID of a file in - table C, C => ID of the file that was used as input to create - the intermediary file, C ID of the output file, C => ID of this file in - table C, C => value of column C in table C for the file with the given - ID, C value of column C for the intermediary file and +RETURNS: + - an array of hash references. Each hash has seven keys: C => ID of a file in + table C, C => ID of the file that was used as input to create + the intermediary file, C ID of the output file, C => ID of this file in + table C, C => value of column C in table C for the file with the given + ID, C value of column C for the intermediary file and C => absolute path of the file with the given ID. =cut sub getIntermediaryFilesRef { my($dbh, $filesRef, $tarchiveID, $dataDirBasePath, $scanTypesToDeleteRef, $optionsRef) = @_; - + return [] if !defined $tarchiveID; my $fileBaseName = $optionsRef->{'BASENAME'}; @@ -1000,13 +1000,13 @@ sub getIntermediaryFilesRef { # This should get all files in table files_intermediary that are tied # indirectly to the tarchive with ID $tarchiveId # Note that there can be multiple entries in files_intermediary that have - # the same Output_FileID: consequently, this select statement can return + # the same Output_FileID: consequently, this select statement can return # entries with identical FileID and File values (but different IntermedID) my $mriScanTypeJoin = @$scanTypesToDeleteRef - ? 'JOIN mri_scan_type mst ON (f2.AcquisitionProtocolID=mst.ID) ' + ? 'JOIN mri_scan_type mst ON (f2.MriScanTypeID=mst.MriScanTypeID) ' : ''; my $mriScanTypeAnd = @$scanTypesToDeleteRef - ? sprintf(' AND mst.Scan_type IN (%s) ', join(',', ('?') x @$scanTypesToDeleteRef)) + ? sprintf(' AND mst.MriScanTypeName IN (%s) ', join(',', ('?') x @$scanTypesToDeleteRef)) : ''; my($query, @queryArgs); @@ -1024,10 +1024,10 @@ sub getIntermediaryFilesRef { . $mriScanTypeAnd . ') ' . 'AND BINARY SUBSTRING_INDEX(f.File, "/", -1) = ?'; - @queryArgs = ($tarchiveID, @$scanTypesToDeleteRef, $fileBaseName); + @queryArgs = ($tarchiveID, @$scanTypesToDeleteRef, $fileBaseName); # Either -basename was not used or it was used but nothing matched in table files. } else { - my $fileBaseNameAnd = $fileBaseName ne '' + my $fileBaseNameAnd = $fileBaseName ne '' ? ' AND BINARY SUBSTRING_INDEX(f2.File, "/", -1) = ?' : ''; $query = 'SELECT fi.IntermedID, fi.Input_FileID, fi.Output_FileID, f.FileID, f.File, f.SourceFileID ' @@ -1041,19 +1041,19 @@ sub getIntermediaryFilesRef { . $mriScanTypeAnd . $fileBaseNameAnd . ')'; - @queryArgs = ($tarchiveID, @$scanTypesToDeleteRef); + @queryArgs = ($tarchiveID, @$scanTypesToDeleteRef); push(@queryArgs, $fileBaseName) if $fileBaseName ne ''; } - + my $intermedFilesRef = $dbh->selectall_arrayref($query, { Slice => {} }, @queryArgs); # Set full path of every file foreach(@$intermedFilesRef) { - $_->{'FullPath'} = $_->{'File'} =~ /^\// + $_->{'FullPath'} = $_->{'File'} =~ /^\// ? $_->{'File'} : "$dataDirBasePath/$_->{'File'}"; - } - - return $intermedFilesRef; + } + + return $intermedFilesRef; } =pod @@ -1070,8 +1070,8 @@ =head3 getParameterFilesRef($dbh, $filesRef, $tarchiveID, $dataDirBasePath, $sca - $scanTypesToDeleteRef: reference to the array that contains the list of scan type names to delete. - $optionsRef: reference on the array of command line options. -RETURNS: - - an array of hash references. Each hash has four keys: C => FileID of a file +RETURNS: + - an array of hash references. Each hash has four keys: C => FileID of a file in table C, C => value of column C in table C for the file with the given ID, C => name of the parameter and C => absolute path of the file with the given ID. @@ -1079,14 +1079,14 @@ =head3 getParameterFilesRef($dbh, $filesRef, $tarchiveID, $dataDirBasePath, $sca =cut sub getParameterFilesRef { my($dbh, $filesRef, $tarchiveID, $dataDirBasePath, $scanTypesToDeleteRef, $optionsRef) = @_; - + return [] if !defined $tarchiveID; my $fileBaseName = $optionsRef->{'BASENAME'}; my $mriScanTypeJoin = @$scanTypesToDeleteRef - ? 'JOIN mri_scan_type mst ON (mst.ID=f.AcquisitionProtocolID) ' : ''; + ? 'JOIN mri_scan_type mst ON (mst.MriScanTypeID=f.MriScanTypeID) ' : ''; my $mriScanTypeAnd = @$scanTypesToDeleteRef - ? sprintf('AND mst.Scan_type IN (%s) ', join(',', ('?') x @$scanTypesToDeleteRef)) : ''; + ? sprintf('AND mst.MriScanTypeName IN (%s) ', join(',', ('?') x @$scanTypesToDeleteRef)) : ''; # If -basename was used but nothing matched in tables files and files_intermediary # Try finding a file in parameter_file that matches the -basename argument @@ -1114,10 +1114,10 @@ sub getParameterFilesRef { . ' AND BINARY SUBSTRING_INDEX(pf.Value, "/", -1) = ?'; push(@queryArgs, $tarchiveID, @$scanTypesToDeleteRef, $fileBaseName); } else { - my $fileBaseNameAnd = $fileBaseName ne '' + my $fileBaseNameAnd = $fileBaseName ne '' ? ' AND BINARY SUBSTRING_INDEX(f.File, "/", -1) = ?' : ''; - + $query = 'SELECT pf.ParameterFileID, f.FileID, pf.Value, pt.Name FROM parameter_file pf ' . 'JOIN files f ON (f.FileID=pf.FileID) ' . 'JOIN parameter_type pt ON (pf.ParameterTypeID=pt.ParameterTypeID) ' @@ -1138,11 +1138,11 @@ sub getParameterFilesRef { . 'WHERE 1=1 ' . $mriScanTypeAnd . " $fileBaseNameAnd"; - + push(@queryArgs, $tarchiveID, @$scanTypesToDeleteRef); push(@queryArgs, $fileBaseName) if $fileBaseName ne ''; } - + # If there are intermediary files and we do not want to keep the defaced files # then we have to add to the query the records in table parameter_file that are # associated to the files in files_intermediary. Note that some of these files might @@ -1162,7 +1162,7 @@ sub getParameterFilesRef { my $parametersRef = $dbh->selectall_arrayref( $query, { Slice => {} }, @queryArgs ); - + # Set full path of every file foreach my $f (@$parametersRef) { # If the parameter is not a file @@ -1173,7 +1173,7 @@ sub getParameterFilesRef { $f->{'FullPath'} = $f->{'Value'} =~ /^\// ? $f->{'Value'} : sprintf("%s/%s", $fileBasePath, $f->{'Value'}); } - + return $parametersRef; } @@ -1181,7 +1181,7 @@ sub getParameterFilesRef { =head3 getMriProtocolViolatedScansFilesRef($dbh, $tarchiveID, $dataDirBasePath, $scanTypesToDeleteRef, $optionsRef) -Get the absolute paths of all the files associated to a DICOM archive that are listed in +Get the absolute paths of all the files associated to a DICOM archive that are listed in table C. INPUTS: @@ -1191,16 +1191,16 @@ =head3 getMriProtocolViolatedScansFilesRef($dbh, $tarchiveID, $dataDirBasePath, - $scanTypesToDeleteRef: reference to the array that contains the list of scan type names to delete. - $optionsRef: reference on the array of command line options. -RETURNS: +RETURNS: - an array of hash references. Each hash has three keys: C => ID of the record in table - C, C => value of column C in table + C, C => value of column C in table C for the MINC file found and C => absolute path of the MINC file found. =cut sub getMriProtocolViolatedScansFilesRef { my($dbh, $tarchiveID, $dataDirBasePath, $scanTypesToDeleteRef, $optionsRef) = @_; - + # Return a reference to the empty array if: # 1. The upload(s) do not have a tarchiveID, which means that no MINC file # were produced and consequently there cannot be any protocol violations @@ -1221,18 +1221,18 @@ sub getMriProtocolViolatedScansFilesRef { # Set full path of every file foreach(@$filesRef) { - $_->{'FullPath'} = $_->{'minc_location'} =~ /^\// + $_->{'FullPath'} = $_->{'minc_location'} =~ /^\// ? $_->{'minc_location'} : "$dataDirBasePath/$_->{'minc_location'}"; - } - - return $filesRef; + } + + return $filesRef; } =pod =head3 getMriViolationsLogFilesRef($dbh, $tarchiveID, $dataDirBasePath, $scanTypesToDeleteRef, $optionsRef) -Get the absolute paths of all the files associated to an archive that are listed in +Get the absolute paths of all the files associated to an archive that are listed in table C. INPUTS: @@ -1242,15 +1242,15 @@ =head3 getMriViolationsLogFilesRef($dbh, $tarchiveID, $dataDirBasePath, $scanTyp - $scanTypesToDeleteRef: reference to the array that contains the list of scan type names to delete. - $optionsRef: reference on the array of command line options. -RETURNS: - an array of hash references. Each hash has three keys: C => ID of the record in table +RETURNS: + an array of hash references. Each hash has three keys: C => ID of the record in table C, C => value of column C for the MINC file found in table C and C => absolute path of the MINC file. =cut sub getMriViolationsLogFilesRef { my($dbh, $tarchiveID, $dataDirBasePath, $scanTypesToDeleteRef, $optionsRef) = @_; - + return [] if !defined $tarchiveID; my $fileBaseName = $optionsRef->{'BASENAME'}; @@ -1262,9 +1262,9 @@ sub getMriViolationsLogFilesRef { $query = 'SELECT LogID, mvl.MincFile ' . 'FROM mri_violations_log mvl ' . 'JOIN files f ON (f.File COLLATE utf8_bin = mvl.MincFile) ' - . 'JOIN mri_scan_type mst ON (f.AcquisitionProtocolID = mst.ID) ' + . 'JOIN mri_scan_type mst ON (f.MriScanTypeID = mst.MriScanTypeID) ' . 'WHERE TarchiveID = ? ' - . 'AND mst.Scan_type IN(' + . 'AND mst.MriScanTypeName IN(' . join(',', ('?') x @$scanTypesToDeleteRef) . ')' . $fileBaseNameAnd; @@ -1272,25 +1272,25 @@ sub getMriViolationsLogFilesRef { $query = 'SELECT LogID, MincFile FROM mri_violations_log WHERE TarchiveID = ?' . $fileBaseNameAnd; } - + my @queryArgs = ($tarchiveID, @$scanTypesToDeleteRef); push(@queryArgs, $fileBaseName) if $fileBaseName ne ''; my $filesRef = $dbh->selectall_arrayref($query, { Slice => {} }, @queryArgs); # Set full path of every file foreach(@$filesRef) { - $_->{'FullPath'} = $_->{'MincFile'} =~ /^\// + $_->{'FullPath'} = $_->{'MincFile'} =~ /^\// ? $_->{'MincFile'} : "$dataDirBasePath/$_->{'MincFile'}"; - } + } - return $filesRef; + return $filesRef; } =pod =head3 getMRICandidateErrorsFilesRef($dbh, $tarchiveID, $dataDirBasePath, $scanTypeToDeleteRef, $optionsRef) -Get the absolute paths of all the files associated to a DICOM archive that are listed in +Get the absolute paths of all the files associated to a DICOM archive that are listed in table C. INPUTS: @@ -1300,21 +1300,21 @@ =head3 getMRICandidateErrorsFilesRef($dbh, $tarchiveID, $dataDirBasePath, $scanT - $scanTypesToDeleteRef: reference to the array that contains the list of scan type names to delete. - $optionsRef: reference on the array of command line options. -RETURNS: - - an array of hash references. Each hash has three keys: C => ID of the record in the - table, C => value of column C for the MINC file found in table +RETURNS: + - an array of hash references. Each hash has three keys: C => ID of the record in the + table, C => value of column C for the MINC file found in table C and C => absolute path of the MINC file. =cut sub getMRICandidateErrorsFilesRef { my($dbh, $tarchiveID, $dataDirBasePath, $scanTypeToDeleteRef, $optionsRef) = @_; - + return [] if !defined $tarchiveID || @$scanTypeToDeleteRef; # Get file path of each files in files directly tied # to $tarchiveId my $fileBaseName = $optionsRef->{'BASENAME'}; - my $fileBaseNameAnd = $fileBaseName ne '' + my $fileBaseNameAnd = $fileBaseName ne '' ? ' AND BINARY SUBSTRING_INDEX(MincFile, "/", -1) = ?' : ''; my $query = 'SELECT ID, MincFile FROM MRICandidateErrors WHERE TarchiveID = ?' @@ -1325,11 +1325,11 @@ sub getMRICandidateErrorsFilesRef { # Set full path of every file foreach(@$filesRef) { - $_->{'FullPath'} = $_->{'MincFile'} =~ /^\// + $_->{'FullPath'} = $_->{'MincFile'} =~ /^\// ? $_->{'MincFile'} : "$dataDirBasePath/$_->{'MincFile'}"; - } - - return $filesRef; + } + + return $filesRef; } =pod @@ -1358,9 +1358,9 @@ sub getBidsExportFilesRef { my $fileBaseName = $optionsRef->{'BASENAME'}; my $mriScanTypeJoin = @$scanTypesToDeleteRef - ? 'JOIN mri_scan_type mst ON (mst.ID=f.AcquisitionProtocolID) ' : ''; + ? 'JOIN mri_scan_type mst ON (mst.MriScanTypeID=f.MriScanTypeID) ' : ''; my $mriScanTypeAnd = @$scanTypesToDeleteRef - ? sprintf('AND mst.Scan_type IN (%s) ', join(',', ('?') x @$scanTypesToDeleteRef)) : ''; + ? sprintf('AND mst.MriScanTypeName IN (%s) ', join(',', ('?') x @$scanTypesToDeleteRef)) : ''; # If -basename was used but nothing matched in tables files and files_intermediary # Try finding a file in parameter_file that matches the -basename argument @@ -1539,41 +1539,41 @@ sub updateBidsScansTsvFile { =head3 setFileExistenceStatus($filesRef) -Checks the list of all the files related to the upload(s) that were found in the database and +Checks the list of all the files related to the upload(s) that were found in the database and determine whether they exist or not on the file system. INPUTS: - $filesRef: reference to the array that contains the file informations for all the files that are associated to the upload(s) passed on the command line. - - + + RETURNS: - Reference on the list of files that do not exist on the file system. - + =cut sub setFileExistenceStatus { my($filesRef) = @_; - + my %missingFiles; foreach my $t (@PROCESSED_TABLES) { foreach my $f (@{ $filesRef->{$t} }) { $f->{'Exists'} = undef, next if !$f->{'FullPath'}; - + # The -f test is to ensure $f->{'FullPath'} is an actual file # and not a directory. If $f->{'FullPath'} is a directory, it # means that it has an incorrect/invalid value and should not be backed up # later on. $f->{'Exists'} = -f $f->{'FullPath'} && -e $f->{'FullPath'}; - + # A file is only considered "missing" if it is expected to be on the file # system but is not. There are some file paths refered to in the database that # are expected to refer to files that have been moved or deleted and consequently - # they are not expected to be on the file system at the specified location (e.g. + # they are not expected to be on the file system at the specified location (e.g. # ArchiveLocation for an upload on which the MRI pipeline was successfully run). $missingFiles{ $f->{'FullPath'} } = 1 if !$f->{'Exists'} && &shouldExist($t, $f); } } - + return [ keys %missingFiles ]; } @@ -1586,17 +1586,17 @@ =head3 shouldExist($table, $fileRef) INPUTS: - $table: name of the table in which the file path was found. - $fileRef: reference to the array that contains the file information for a given file. - + RETURNS: - 0 or 1 depending on whether the file should exist or not. - + =cut sub shouldExist { my($table, $fileRef) = @_; - + return 0 if $table eq 'mri_upload' && defined $fileRef->{'InsertionComplete'} && $fileRef->{'InsertionComplete'} == 1; - - + + return 1; } @@ -1610,36 +1610,36 @@ =head3 backupFiles($filesRef, $scanTypesToDeleteRef, $optionsRef) that are associated to the upload(s) passed on the command line. - $scanTypesToDeleteRef: reference to the array that contains the list of scan type names to delete. - $optionsRef: reference on the hash array of command line options. - + RETURNS: - The number of files backed up. =cut - + sub backupFiles { my($filesRef, $scanTypesToDeleteRef, $optionsRef) = @_; - + my $hasSomethingToBackup = 0; foreach my $t (@PROCESSED_TABLES) { foreach my $f (@{ $filesRef->{$t} }) { last if $hasSomethingToBackup; - + # Not all entries in paramter_file are files... next if $t eq 'parameter_file' && !grep($f->{'Name'} eq $_, FILE_PARAMETER_NAMES); - + # If the file is not going to be deleted, do not back it up next unless &shouldDeleteFile($t, $f, $scanTypesToDeleteRef, $optionsRef->{'KEEP_DEFACED'}, $optionsRef->{'BASENAME'}); $hasSomethingToBackup = 1 if $f->{'Exists'} } } - + if(!$hasSomethingToBackup) { print "No files to backup.\n"; return 0; } - + # Create a temporary file that will list the absolute paths of all - # files to backup (archive). + # files to backup (archive). my $nbToBackUp = 0; my($fh, $tmpFileName) = tempfile("$0.filelistXXXX", UNLINK => 1); foreach my $t (@PROCESSED_TABLES) { @@ -1657,16 +1657,16 @@ sub backupFiles { print $fh "$filesRef->{'bids_export_files_scans_tsv'}{'scans_tsv_file_path'}"; } close($fh); - + # Put all files in a big compressed tar ball my $filesBackupPath = $optionsRef->{'BACKUP_PATH'} . '.tar'; print "\nBacking up files related to the upload(s) to delete...\n"; if(system('tar', 'cvf', $filesBackupPath, '--absolute-names', '--files-from', $tmpFileName)) { print STDERR "backup command failed: $!\n"; exit $NeuroDB::ExitCodes::PROGRAM_EXECUTION_FAILURE; - } + } print "\n"; - + return $nbToBackUp; } @@ -1682,30 +1682,30 @@ =head3 shouldDeleteFile($table, $fileRef, $scanTypesToDeleteRef, $keepDefaced, $ - $scanTypesToDeleteRef: reference to the array that contains the list of scan type names to delete. - $keepDefaced: whether the defaced files should be kept or not. - $fileBaseName: value of option -basename (or '' if -basename was not used). - + RETURNS: - 0 or 1 depending on whether the file should be deleted or not. - + =cut sub shouldDeleteFile { my($table, $fileRef, $scanTypesToDeleteRef, $keepDefaced, $fileBaseName) = @_; - + # Do not delete/backup the tarchive if any of these is true: # 1. Specific scan types are deleted. - # 2. The defaced files are kept (i.e @$scanTypesToDeleteRef is filled with the + # 2. The defaced files are kept (i.e @$scanTypesToDeleteRef is filled with the # modalities_to_deface Config values). # 3. Option -basename was used. return 0 if $table eq 'tarchive' && (@$scanTypesToDeleteRef || ($fileBaseName ne '')); - + # If the defaced files are kept, the entries in files_intermediary are deleted - # and backed up but the corresponding entries in table files are not. + # and backed up but the corresponding entries in table files are not. return 0 if $table eq 'files_intermediary' && $keepDefaced; - + # If the MRI pipeline was successfully run on the upload, the file has moved # so do not attempt to delete it return 0 if $table eq 'mri_upload' && ($fileRef->{'InsertionComplete'} == 1 || ($fileBaseName ne '')); - + return 1; } @@ -1713,12 +1713,12 @@ sub shouldDeleteFile { =head3 deleteUploadsInDatabase($dbh, $filesRef, $scanTypesToDeleteRef, $optionsRef) -This method deletes all information in the database associated to the given upload(s)/scan type combination. +This method deletes all information in the database associated to the given upload(s)/scan type combination. More specifically, it deletes records from tables C, C, C C, C, C, C, C -C, C, C, C and C +C, C, C, C and C (the later is done only if requested). It will also set the C value of the scan's session to 'N' for -each upload that is the last upload tied to that session. All the delete/update operations are done inside a single +each upload that is the last upload tied to that session. All the delete/update operations are done inside a single transaction so either they all succeed or they all fail (and a rollback is performed). INPUTS: @@ -1727,52 +1727,52 @@ =head3 deleteUploadsInDatabase($dbh, $filesRef, $scanTypesToDeleteRef, $optionsR that are associated to the upload(s) passed on the command line. - $scanTypesToDeleteRef: reference to the array that contains the list of scan type names to delete. - $optionsRef: reference on the hash array of the options that were passed on the command line. - + RETURNS: - A reference to a hash with two keys: * SQL_BACKUP_DONE => 1 if this method produced a file containing the SQL statements that restore the database state to what it was before calling this method, 0 otherwise. * NB_RECORDS_DELETED => the number of records effectively deleted by this method. - + =cut sub deleteUploadsInDatabase { my($dbh, $filesRef, $scanTypesToDeleteRef, $optionsRef)= @_; - + # This starts a DB transaction. All operations are delayed until # commit is called $dbh->begin_work; - + my(undef, $tmpSQLFile) = $optionsRef->{'NO_SQL_BK'} ? (undef, undef) : tempfile('sql_backup_XXXX', UNLINK => 1); - + my @IDs = map { $_->{'UploadID'} } @{ $filesRef->{'mri_upload'} }; my $nbRecordsDeleted = 0; if(!@$scanTypesToDeleteRef && $optionsRef->{'BASENAME'} eq '') { $nbRecordsDeleted += &deleteTableData($dbh, 'notification_spool', 'ProcessID', \@IDs, $tmpSQLFile, $optionsRef); } - + # If only specific scan types are targeted for deletion, do not delete the entries in - # tarchive_files and tarchive_series as these are tied to the archive, not the MINC files + # tarchive_files and tarchive_series as these are tied to the archive, not the MINC files if(!@$scanTypesToDeleteRef && $optionsRef->{'BASENAME'} eq '') { - my $IDsRef = &getTarchiveSeriesIDs($dbh, $filesRef); + my $IDsRef = &getTarchiveSeriesIDs($dbh, $filesRef); $nbRecordsDeleted += &deleteTableData($dbh, 'tarchive_files', 'TarchiveSeriesID', $IDsRef, $tmpSQLFile, $optionsRef); - + $nbRecordsDeleted += &deleteTableData($dbh, 'tarchive_series', 'TarchiveSeriesID', $IDsRef, $tmpSQLFile, $optionsRef); } - + @IDs = map { $_->{'ParameterFileID'} } @{ $filesRef->{'parameter_file'} }; $nbRecordsDeleted += &deleteTableData($dbh, 'parameter_file', 'ParameterFileID', \@IDs, $tmpSQLFile, $optionsRef); - + @IDs = map { $_->{'IntermedID'} } @{ $filesRef->{'files_intermediary'} }; $nbRecordsDeleted += &deleteTableData($dbh, 'files_intermediary', 'IntermedID', \@IDs, $tmpSQLFile, $optionsRef); - + # Since all files in files_intermediary are linked to other files in table files, we # have to delete these files first from table files. if(!$optionsRef->{'KEEP_DEFACED'}) { @IDs = map { $_->{'FileID'} } @{ $filesRef->{'files_intermediary'} }; $nbRecordsDeleted += &deleteTableData($dbh, 'files', 'FileID', \@IDs, $tmpSQLFile, $optionsRef); - } else { + } else { &updateFilesIntermediaryTable($dbh, $filesRef, $tmpSQLFile); } @@ -1781,19 +1781,19 @@ sub deleteUploadsInDatabase { @IDs = map { $_->{'FileID'} } @{ $filesRef->{'files'} }; $nbRecordsDeleted += &deleteTableData($dbh, 'files', 'FileID', \@IDs, $tmpSQLFile, $optionsRef); - + @IDs = map { $_->{'ProcessProtocolID'} } @{ $filesRef->{'mri_processing_protocol'} }; $nbRecordsDeleted += &deleteTableData($dbh, 'mri_processing_protocol', 'ProcessProtocolID', \@IDs, $tmpSQLFile, $optionsRef); @IDs = map { $_->{'ID'} } @{ $filesRef->{'mri_protocol_violated_scans'} }; $nbRecordsDeleted += &deleteTableData($dbh, 'mri_protocol_violated_scans', 'ID', \@IDs, $tmpSQLFile, $optionsRef); - + @IDs = map { $_->{'LogID'} } @{ $filesRef->{'mri_violations_log'} }; $nbRecordsDeleted += &deleteTableData($dbh, 'mri_violations_log', 'LogID', \@IDs, $tmpSQLFile, $optionsRef); - + @IDs = map { $_->{'ID'} } @{ $filesRef->{'MRICandidateErrors'} }; $nbRecordsDeleted += &deleteTableData($dbh, 'MRICandidateErrors', 'ID', \@IDs, $tmpSQLFile, $optionsRef); - + $nbRecordsDeleted += &deleteMriParameterForm($dbh, $filesRef->{'mri_upload'}, $tmpSQLFile, $optionsRef) if $optionsRef->{'DELETE_MRI_PARAMETER_FORM'}; # Should check instead if the tarchive is not tied to anything @@ -1805,18 +1805,18 @@ sub deleteUploadsInDatabase { if(!@$scanTypesToDeleteRef && $optionsRef->{'BASENAME'} eq '') { @IDs = map { $_->{'UploadID'} } @{ $filesRef->{'mri_upload'} }; $nbRecordsDeleted += &deleteTableData($dbh, 'mri_upload', 'UploadID', \@IDs, $tmpSQLFile, $optionsRef); - + $nbRecordsDeleted += &deleteTableData($dbh, 'tarchive', 'TarchiveID', [$tarchiveID], $tmpSQLFile, $optionsRef) if defined $tarchiveID; } - + &updateSessionTable($dbh, $filesRef->{'mri_upload'}, $tmpSQLFile) unless @$scanTypesToDeleteRef || $optionsRef->{'BASENAME'} ne ''; - + $dbh->commit; - + # If the SQL restore file should be produced my $sqlBackupDone = 0; if(!$optionsRef->{'NO_SQL_BK'}) { - # If there was actually *something* to restore. In some cases, there + # If there was actually *something* to restore. In some cases, there # is nothing to delete and consequently nothing to restore. if(defined $tmpSQLFile && -s $tmpSQLFile) { my $tarOptions = $optionsRef->{'NO_FILES_BK'} ? 'cf' : 'rf'; @@ -1824,15 +1824,15 @@ sub deleteUploadsInDatabase { print STDERR "Failed to add SQL restore statements file to the backup file: $!\n"; exit $NeuroDB::ExitCodes::PROGRAM_EXECUTION_FAILURE; } - + printf("Added %s to the backup file.\n", SQL_RESTORE_NAME); - + $sqlBackupDone = 1; } else { print "Nothing deleted from the database: skipping creation of the SQL restore file.\n" } - } - + } + return { SQL_BACKUP_DONE => $sqlBackupDone, NB_RECORDS_DELETED => $nbRecordsDeleted @@ -1848,18 +1848,18 @@ =head3 gzipBackupFile($backupPath) INPUTS: - $backupPath: path of the backup file to compress (without the .tar.gz extension). - + =cut sub gzipBackupFile { my($backupPath) = @_; - + # Gzip the backup file my $cmd = sprintf("gzip -f %s.tar", quotemeta($backupPath)); system($cmd) == 0 or die "Failed running command $cmd. Aborting\n"; - + print "Wrote $backupPath.tar.gz\n"; -} +} =pod @@ -1876,19 +1876,19 @@ =head3 updateSessionTable($dbh, $mriUploadsRef, $tmpSQLFile) in the array. The properties stored for each hash are: C, C, C C, C and C. - $tmpSQLFile: path of the SQL file that contains the SQL statements used to restore the deleted records. - + =cut sub updateSessionTable { my($dbh, $mriUploadsRef, $tmpSQLFile) = @_; - - # If any of the uploads to delete is the last upload that was part of the + + # If any of the uploads to delete is the last upload that was part of the # session associated to it, then set the session's 'Scan_done' flag # to 'N'. my @sessionIDs = map { $_->{'SessionID'} } @$mriUploadsRef; @sessionIDs = grep(defined $_, @sessionIDs); - + return if !@sessionIDs; - + my $query = "UPDATE session s SET Scan_done = 'N'" . " WHERE s.ID IN (" . join(',', ('?') x @sessionIDs) @@ -1898,7 +1898,7 @@ sub updateSessionTable { if($tmpSQLFile) { # Write an SQL statement to restore the 'Scan_done' column of the deleted uploads # to their appropriate values. This statement needs to be after the statement that - # restores table mri_upload (at the end of the file is good enough). + # restores table mri_upload (at the end of the file is good enough). open(SQL, ">>$tmpSQLFile") or die "Cannot append text to file $tmpSQLFile: $!. Aborting.\n"; print SQL "\n\n"; print SQL "UPDATE session s SET Scan_done = 'Y'" @@ -1914,7 +1914,7 @@ sub updateSessionTable { =head3 updateFilesIntermediaryTable($dbh, $filesRef, $tmpSQLFile) Sets the C and C columns of all the defaced files to C<$tarchiveID> and C -respectively. The script also adds an SQL statement in the SQL file whose path is passed as argument to +respectively. The script also adds an SQL statement in the SQL file whose path is passed as argument to restore the state that the defaced files in the C table had before the deletions. INPUTS: @@ -1922,11 +1922,11 @@ =head3 updateFilesIntermediaryTable($dbh, $filesRef, $tmpSQLFile) - $filesRef: reference to the array that contains the file informations for all the files that are associated to the upload(s) passed on the command line. - $tmpSQLFile: path of the SQL file that contains the SQL statements used to restore the deleted records. - + =cut sub updateFilesIntermediaryTable { my($dbh, $filesRef, $tmpSQLFile) = @_; - + my $tarchiveID = $filesRef->{'tarchive'}->[0]->{'TarchiveID'}; my %defacedFiles = map { $_->{'FileID'} => $_->{'SourceFileID'} } @{ $filesRef->{'files_intermediary'} }; if(%defacedFiles) { @@ -1950,7 +1950,7 @@ sub updateFilesIntermediaryTable { =head3 deleteMriParameterForm($dbh, $mriUploadsRef, $tmpSQLFile, $optionsRef) Delete the entries in C (and associated C entry) for the upload(s) passed on the -command line. The script also adds an SQL statement in the SQL file whose path is passed as argument to +command line. The script also adds an SQL statement in the SQL file whose path is passed as argument to restore the state that the C and C tables had before the deletions. INPUTS: @@ -1964,22 +1964,22 @@ =head3 deleteMriParameterForm($dbh, $mriUploadsRef, $tmpSQLFile, $optionsRef) RETURNS: - The numbers of records deleted as a result of this operation. - + =cut sub deleteMriParameterForm { my($dbh, $mriUploadsRef, $tmpSQLFile, $optionsRef) = @_; - + my @uploadIDs = map { $_->{'UploadID'} } @$mriUploadsRef; my $query = "SELECT f.CommentID FROM flag f " . "JOIN session s ON (s.ID=f.SessionID) " . "JOIN mri_upload mu ON (s.ID=mu.SessionID) " . "WHERE f.Test_name='mri_parameter_form' " - . "AND mu.UploadID IN ( " + . "AND mu.UploadID IN ( " . join(',', ('?') x @uploadIDs) . ") "; my $rowsRef = $dbh->selectall_arrayref($query, { Slice => {} }, @uploadIDs); my @commentIDs = map { $_->{'CommentID'} } @$rowsRef; - + return if !@commentIDs; my $nbRecordsDeleted = 0; @@ -2003,20 +2003,20 @@ =head3 deleteUploadsOnFileSystem($filesRef, $scanTypesToDeleteRef, $keepDefaced, - $scanTypesToDeleteRef: reference to the array that contains the list of scan type names to delete. - $keepDefaced: whether the defaced files should be kept or not. - $fileBaseName: value of option -basename (or '' if option was not used). - + =cut sub deleteUploadsOnFileSystem { my($filesRef, $scanTypesToDeleteRef, $keepDefaced, $fileBaseName) = @_; - + my %deletedFile; foreach my $t (@PROCESSED_TABLES) { foreach my $f (@{ $filesRef->{$t} }) { next if !shouldDeleteFile($t, $f, $scanTypesToDeleteRef, $keepDefaced, $fileBaseName); - + next if !$f->{'Exists'}; - + next if $deletedFile{ $f->{'FullPath'} }; - + NeuroDB::MRI::deleteFiles($f->{'FullPath'}); $deletedFile{ $f->{'FullPath'} } = 1; } @@ -2035,29 +2035,29 @@ =head3 getTypesToDelete($dbh, $scanTypeList, $keepDefaced) - $dbh: database handle. - $scanTypeList: comma separated string of scan type names. - $keepDefaced: whether the defaced files should be kept or not. - + RETURNS: - A reference on a hash of the names of the scan types to delete: key => scan type name, value => 1 or 0 depending on whether the name is valid or not. =cut sub getScanTypesToDelete { my($dbh, $scanTypeList, $keepDefaced) = @_; - + return () if !defined $scanTypeList && !$keepDefaced; - + if($keepDefaced) { my @scanTypes = $configOB->getModalitiesToDeface(); return (map { $_ => 1 } @scanTypes); } - + my %types = map { $_=> 1 } split(/,/, $scanTypeList); - - my $query = 'SELECT Scan_type, ID FROM mri_scan_type ' - . 'WHERE Scan_type IN (' + + my $query = 'SELECT MriScanTypeName, MriScanTypeID FROM mri_scan_type ' + . 'WHERE MriScanTypeName IN (' . join(',', ('?') x keys %types) . ')'; - my $validTypesRef = $dbh->selectall_hashref($query, 'Scan_type', undef, keys %types); - + my $validTypesRef = $dbh->selectall_hashref($query, 'MriScanTypeName', undef, keys %types); + return map { $_ => defined $validTypesRef->{$_} } keys %types; } @@ -2071,22 +2071,22 @@ =head3 getTarchiveSeriesIDs($dbh, $filesRef) - $dbh: database handle. - $filesRef: reference to the array that contains the file informations for all the files that are associated to the upload(s) passed on the command line. - + RETURNS: - A reference on an array containing the C to delete. - + =cut sub getTarchiveSeriesIDs { my($dbh, $filesRef) = @_; - + return [] unless @{ $filesRef->{'files'} }; - + my @fileID = map { $_->{'FileID'} } @{ $filesRef->{'files'} }; my $query = 'SELECT tf.TarchiveSeriesID ' . 'FROM tarchive_files tf ' . 'JOIN tarchive_series ts ON (tf.TarchiveSeriesID=ts.TarchiveSeriesID) ' . "JOIN files f ON (ts.SeriesUID=f.SeriesUID AND ABS(f.EchoTime*1000 - ts.EchoTime) < $FLOAT_EQUALS_THRESHOLD) " - . 'WHERE f.FileID IN (' + . 'WHERE f.FileID IN (' . join(',', ('?') x @fileID) . ')'; my $tarchiveFilesRef = $dbh->selectall_arrayref($query, { Slice => {} }, @fileID); @@ -2097,7 +2097,7 @@ sub getTarchiveSeriesIDs { =head3 deleteTableData($dbh, $table, $key, $keyValuesRef, $tmpSQLBackupFile, $optionsRef) Deletes records from a database table and adds in a file the SQL statements that allow rewriting the -records back in the table. +records back in the table. INPUTS: @@ -2110,13 +2110,13 @@ =head3 deleteTableData($dbh, $table, $key, $keyValuesRef, $tmpSQLBackupFile, $op RETURNS: - The number of records deleted. - + =cut sub deleteTableData { my($dbh, $table, $key, $keyValuesRef, $tmpSQLBackupFile, $optionsRef) = @_; - + return 0 unless @$keyValuesRef; - + my $query = "DELETE FROM $table WHERE $key IN(" . join(',', ('?') x @$keyValuesRef) . ')'; @@ -2143,7 +2143,7 @@ =head3 updateSQLBackupFile($tmpSQLBackupFile, $table, $key, $keyValuesRef, $opti =cut sub updateSQLBackupFile { my($tmpSQLBackupFile, $table, $key, $keyValuesRef, $optionsRef) = @_; - + # Make sure all keys are quoted before using them in the Unix command my %quotedKeys; foreach my $k (@$keyValuesRef) { @@ -2151,12 +2151,12 @@ sub updateSQLBackupFile { $quotedKey = "\"$quotedKey\"" if $quotedKey =~ /\D/; $quotedKeys{$quotedKey} = 1; } - + # Read the current contents of the backup file open(SQL, "<$tmpSQLBackupFile") or die "Cannot read $tmpSQLBackupFile: $!\n"; my @lines = ; close(SQL); - + # Run the mysqldump command for the current table and store the # result in $tmpSqlBackupFile (overwrite contents) # Note that --defaults-extra-file has to be the first option (it it is provided) @@ -2215,28 +2215,28 @@ =head3 getInvalidDefacedFiles($dbh, $filesRef, $scanTypesToDeleteRef) - $filesRef: reference to the array that contains the information for all the files that are associated to the upload(s) passed on the command line. - $scanTypesToDeleteRef: reference to the array that contains the list of scan type names to delete. - + RETURNS: - The hash of MINC files that were either not defaced (and should have been) or that have more than one processed file associated to them. Key => file path (relative), Value => reference on an array that contains the list of processed files associated to the MINC file (0, 2, 3 or more entries). - + =cut sub getInvalidDefacedFiles { my($dbh, $filesRef, $scanTypesToDeleteRef) = @_; - + my %invalidDefacedFile; foreach my $f (@{ $filesRef->{'files'} }) { my @processedFiles = grep($_->{'Input_FileID'} == $f->{'FileID'}, @{ $filesRef->{'files_intermediary'} }); $invalidDefacedFile{ $f->{'File'} } = [] if !@processedFiles; - + my @filesNotDefaced = grep( $_->{'File'} !~ /-defaced_\d+.mnc$/, @processedFiles); $invalidDefacedFile{ $f->{'File'} } = [ map { $_->{'File'} } @filesNotDefaced ] if @filesNotDefaced; - + # otherwise it means that @processedFiles contains one and only one file # that ends with '-defaced', which is what is expected } - + return \%invalidDefacedFile; } diff --git a/tools/get_dicom_files.pl b/tools/get_dicom_files.pl index 68ac17b64..0c992628c 100755 --- a/tools/get_dicom_files.pl +++ b/tools/get_dicom_files.pl @@ -1,4 +1,4 @@ -#!/usr/bin/perl -w +#!/usr/bin/perl -w use strict; =pod @@ -9,7 +9,7 @@ =head1 NAME =head1 SYNOPSIS -perl get_dicom_files.pl [-name patient_name_patterns] [-type scan_type_patterns] [-outdir tmp_dir] [-outfile tarBasename] +perl get_dicom_files.pl [-name patient_name_patterns] [-type scan_type_patterns] [-outdir tmp_dir] [-outfile tarBasename] [-id candid|pscid|candid_pscid|pscid_candid] -profile profile Available options are: @@ -17,17 +17,17 @@ =head1 SYNOPSIS -profile : name of the config file in C<../dicom-archive/.loris_mri> (typically C) -name : comma separated list of MySQL patterns for the patient names that a DICOM file - has to have in order to be extracted. A DICOM file only has to match one of the - patterns to be extracted. If no pattern is specified, then the patient name is + has to have in order to be extracted. A DICOM file only has to match one of the + patterns to be extracted. If no pattern is specified, then the patient name is not used to determine which DICOM files to extract. This option must be used if no scan type patterns were specified with C<-type> (see below). - + -type : comma separated list of MySQL patterns of the acquisition protocols (scan types names) that a DICOM file has to have in order to be extracted. A DICOM file only has to match one of the patterns to be extracted. If no pattern is specified, then the scan type name is not used to determine which DICOM files to extract. This option must be used if no patient name patterns were specified via C<-names> (see above). - + -outdir : full path of the root directory in which the script will temporarily extract the DICOM files before building the final C<.tar.gz> file. The files will be extracted in directory C<< /get_dicom_files.pl. >>. For example with @@ -38,10 +38,10 @@ =head1 SYNOPSIS directory that resides on an NFS mounted file system. Failure to do so might result in C failing. Note that this argument has no impact on the location of the final C<.tar.gz> file. Use option C<-outfile> to control that. - --outfile : basename of the final C file to produce, in the current directory (defaults to + +-outfile : basename of the final C file to produce, in the current directory (defaults to C). - + -id : how to name the subdirectory identifying the candidate to which the DICOM files belong: pscid, candid, pscid_candid or candid_pscid (defaults to candid) @@ -49,16 +49,16 @@ =head1 DESCRIPTION This script first connects to the database to build the list of DICOM archives for which the patient names match the list of patterns specified as argument, or all DICOM archives if -no patterns were specified. The script will then examine these DICOM archives and look for the -MINC files whose scan types (acquisition protocol names) match the list of patterns passed as +no patterns were specified. The script will then examine these DICOM archives and look for the +MINC files whose scan types (acquisition protocol names) match the list of patterns passed as argument, or all MINC files for that archive if C<-type> was not used. It then extracts the DICOM files -associated to each MINC file and writes them in the extraction directory (see C<-outdir> option), in a +associated to each MINC file and writes them in the extraction directory (see C<-outdir> option), in a subdirectory with name C<< ///__> -where C<< >> is the index number of the MINC file to which the DICOMs are associated: -e.g. for file C, the MINC index is 2 (i.e. the second MINC file with +where C<< >> is the index number of the MINC file to which the DICOMs are associated: +e.g. for file C, the MINC index is 2 (i.e. the second MINC file with scan type C). Note that the C subdirectory in the file path can be changed to another identifier with option C<-id>. Finally, a C<.tar.gz> that contains all the DICOM files that were extracted is created. @@ -81,12 +81,12 @@ =head1 DESCRIPTION use Getopt::Tabular; &Getopt::Tabular::AddPatternType( - "candidate_identifier", - "pscid|candid|pscid_candid|candid_pscid", + "candidate_identifier", + "pscid|candid|pscid_candid|candid_pscid", "one of the strings 'pscid', 'candid' , 'pscid_candid' or 'candid_pscid'" ); -# If the absolute value of the difference between two floating +# If the absolute value of the difference between two floating # point numbers is lower than this, the two numbers are considered # equal my $FLOAT_EQUALS_THRESHOLD = 0.00001; @@ -101,15 +101,15 @@ =head1 DESCRIPTION my @opt_table = ( ["-profile", "string" , 1, \$profile, "name of config file in ../dicom-archive/.loris_mri"], - ["-name" , "string" , 1, \$patientNames, + ["-name" , "string" , 1, \$patientNames, "comma-separated list of MySQL patterns for the patient name"], - ["-type" , "string" , 1, \$scanTypes, + ["-type" , "string" , 1, \$scanTypes, "comma-separated list of MySQL patterns for the scan type"], - ["-outdir" , "string" , 1, \$tmpExtractBaseDir, - "base path of the temporary directory where files are extracted"], - ["-outfile", "string" , 1, \$outTarBasename, - "basename of the final .tar.gz file to produce (defaults to 'dicoms.tar.gz')"], - ["-id" , "candidate_identifier", 1, \$candidateIdentifier, + ["-outdir" , "string" , 1, \$tmpExtractBaseDir, + "base path of the temporary directory where files are extracted"], + ["-outfile", "string" , 1, \$outTarBasename, + "basename of the final .tar.gz file to produce (defaults to 'dicoms.tar.gz')"], + ["-id" , "candidate_identifier", 1, \$candidateIdentifier, "how to name the subdirectory identifying the candidate to which the DICOM files belong:" . "one of pscid, candid, pscid_candid or candid_pscid (defaults to candid)"] ); @@ -120,22 +120,22 @@ =head1 DESCRIPTION This script first connects to the database to build the list of DICOM archives for which the patient names match the list of patterns specified as argument or all DICOM archives if -no patterns were specified. The script will then examine these DICOM archives and look for the -MINC files whose scan types (acquisition protocol names) match the list of patterns passed as +no patterns were specified. The script will then examine these DICOM archives and look for the +MINC files whose scan types (acquisition protocol names) match the list of patterns passed as argument, or all MINC files for that archive if -type was not used. It then extracts the DICOM files -associated to each MINC file and writes them in the extraction directory (see -outdir option), in a +associated to each MINC file and writes them in the extraction directory (see -outdir option), in a subdirectory with name ///__ where is the index number of the MINC file to which the DICOMs are associated: e.g. for file 'loris_300001_V4_DtiSA_002.mnc', the MINC index is 2 (i.e. the second MINC file with scan type -'DtiSA'). Note that the dccid subdirectory in the file path can be changed to another identifier with +'DtiSA'). Note that the dccid subdirectory in the file path can be changed to another identifier with option -id. Finally, a '.tar.gz' that contains all the DICOM files that were extracted is created. HELP my $Usage = <fetchall_arrayref }) { my($pscid, $candid, $visitLabel, $dateAcquired, $archiveLocation, $tarchiveId) = @$tarchiveRowRef; - + my($innerTar) = $archiveLocation =~ /DCM_\d+-\d+-\d+_([^\/]+)\.tar$/; $innerTar .= '.tar.gz'; - - # Extract only the .tar.gz archive from the main archive (ignore the + + # Extract only the .tar.gz archive from the main archive (ignore the # meta data and log file) print "Extracting $innerTar in $tmpExtractDir..."; my $cmd = sprintf( @@ -285,17 +285,17 @@ =head1 DESCRIPTION ); system($cmd) == 0 or die "Extraction of '$innerTar' in '$tmpExtractDir' failed: $?"; print "done\n"; - - # Fetch all the MINC files created out of the DICOM archive whose + + # Fetch all the MINC files created out of the DICOM archive whose # acquisition protocols match the scan types # of interest $query = "SELECT DISTINCT f.File, tf.FileName, ts.SeriesDescription, tf.md5sum " . "FROM files f " - . "JOIN mri_scan_type mst ON (mst.ID=f.AcquisitionProtocolID) " + . "JOIN mri_scan_type mst ON (mst.MriScanTypeID=f.MriScanTypeID) " . "JOIN tarchive_series ts ON (f.SeriesUID=ts.SeriesUID AND ABS(f.EchoTime*1000 - ts.EchoTime) < $FLOAT_EQUALS_THRESHOLD) " . "JOIN tarchive_files tf USING (TarchiveSeriesID) " . "WHERE f.TarchiveSource = ?"; - my @where = map { "mst.Scan_type LIKE ?" } @scanTypes; + my @where = map { "mst.MriScanTypeName LIKE ?" } @scanTypes; $query .= sprintf(" AND (%s)", join(" OR ", @where)) if @where; $sth = $dbh->prepare($query); $sth->execute($tarchiveId, @scanTypes); @@ -313,7 +313,7 @@ =head1 DESCRIPTION foreach my $fileRowRef (@{ $sth->fetchall_arrayref }) { my($mincFile, $dicomFilename, $seriesDescription, $md5sum) = @$fileRowRef; chomp $dicomFilename; - + $filesRef->{$mincFile}->{'SeriesDescription'} = $seriesDescription; $filesRef->{$mincFile}->{'DICOM'} = [] unless defined $filesRef->{$mincFile}->{'DICOM'}; push(@{ $filesRef->{$mincFile}->{'DICOM'} }, $dicomFilename); @@ -321,7 +321,7 @@ =head1 DESCRIPTION $wantedMd5Sum{$dicomFilename}{$mincFile} = [] unless defined $wantedMd5Sum{$dicomFilename}{$mincFile}; push(@{ $wantedMd5Sum{$dicomFilename}{$mincFile} }, $md5sum); } - + # Foreach MINC file extract the set of DICOM files # that were used to produce it (found above) foreach my $mincFile (keys %$filesRef) { @@ -330,8 +330,8 @@ =head1 DESCRIPTION my($mincFileBaseName) = $mincFile =~ /\/([^\/]+).mnc$/; my $fileList = "$tmpExtractDir/$mincFileBaseName.dicom"; open(FILE_LIST, ">$fileList") or die "Cannot write file $fileList: $!\n"; - - # This hash will contain the base names and relative path of all the DICOM files that were extracted. + + # This hash will contain the base names and relative path of all the DICOM files that were extracted. my %extractedFiles; my $tarCmd = sprintf("tar ztf %s/%s", quotemeta($tmpExtractDir), quotemeta($innerTar)); open(LIST_TAR_CONTENT, "$tarCmd|") or die "Cannot run command $tarCmd: $?\n"; @@ -341,7 +341,7 @@ =head1 DESCRIPTION if(grep($_ eq $dicomFileBaseName, @{ $filesRef->{$mincFile}->{'DICOM'} })) { print FILE_LIST "$_\n"; - # Store the paths of all files to extract. The files will be stored in a hash: + # Store the paths of all files to extract. The files will be stored in a hash: # key -> file basename, value -> file path relative to $dicomFilesExtractDir $extractedFiles{$dicomFileBaseName} = [] unless defined $extractedFiles{$dicomFileBaseName}; push(@{ $extractedFiles{$dicomFileBaseName} }, $_); @@ -349,26 +349,26 @@ =head1 DESCRIPTION } close(LIST_TAR_CONTENT); close(FILE_LIST); - + # Extract all DICOM files in the file list my $dicomFilesExtractDir = "$tmpExtractDir/$mincFileBaseName.extract"; - mkdir $dicomFilesExtractDir or die "Cannot create directory $dicomFilesExtractDir: $!\n"; + mkdir $dicomFilesExtractDir or die "Cannot create directory $dicomFilesExtractDir: $!\n"; $cmd = sprintf( "tar zxf %s/%s -C %s --files-from=%s", quotemeta($tmpExtractDir), quotemeta($innerTar), quotemeta($dicomFilesExtractDir), - quotemeta($fileList) + quotemeta($fileList) ); print "Extracting DICOM files for $mincFileBaseName in $dicomFilesExtractDir..."; - system($cmd) == 0 + system($cmd) == 0 or die "Failed to extract DICOM files for $mincFileBaseName in $dicomFilesExtractDir: $?\n"; print "done.\n"; # Loop over the basenames of all the files we extracted foreach my $dicomFileName (keys %extractedFiles) { # So we want to extract a file with basename $dicomFileName: if there is more than one - # file with that basename in the archive, we'll have to use the MD5Sum to find out + # file with that basename in the archive, we'll have to use the MD5Sum to find out # which one to keep next unless @{ $extractedFiles{$dicomFileName} } > 1; @@ -394,7 +394,7 @@ =head1 DESCRIPTION # $tmpExtractDir/$fileBaseName.dicom my($outSubDir) = $mincFile =~ /_([^_]+_\d+).mnc$/; $outSubDir .= sprintf("_%s", $filesRef->{$mincFile}->{'SeriesDescription'}); - + # Determine the identifier or combination of identifiers to use as part of # the file name my $id; @@ -428,7 +428,7 @@ =head1 DESCRIPTION rmtree($dicomFilesExtractDir); unlink($fileList) or warn "Warning! Could not delete '$tmpExtractDir/$innerTar'\n"; } - + unlink("$tmpExtractDir/$innerTar") or warn "Warning! Could not delete '$tmpExtractDir/$innerTar'\n"; } diff --git a/tools/minc_to_bids_converter.pl b/tools/minc_to_bids_converter.pl index 11158ba3f..79393798a 100755 --- a/tools/minc_to_bids_converter.pl +++ b/tools/minc_to_bids_converter.pl @@ -383,30 +383,30 @@ =head3 getFileList($db_handle, $givenTarchiveID) { "1" => { - 'fileID' => 'FileID value', - 'file' => 'file path', - 'echoTime' => 'Echo Time of the file', - 'AcquisitionProtocolID' => 'Scan type ID', - 'candID' => 'Candidate CandID', - 'sessionID' => 'Session ID', - 'visitLabel' => 'Visit Label', - 'echoNumber' => 'Echo Number of the scan', - 'seriesNumber' => 'Series Number of the scan', - 'imageType' => 'Image Type', - 'lorisScanType' => 'LORIS Scan Type name' + 'fileID' => 'FileID value', + 'file' => 'file path', + 'echoTime' => 'Echo Time of the file', + 'MriScanTypeID' => 'Scan type ID', + 'candID' => 'Candidate CandID', + 'sessionID' => 'Session ID', + 'visitLabel' => 'Visit Label', + 'echoNumber' => 'Echo Number of the scan', + 'seriesNumber' => 'Series Number of the scan', + 'imageType' => 'Image Type', + 'lorisScanType' => 'LORIS Scan Type name' }, "2" => { - 'fileID' => 'FileID value', - 'file' => 'file path', - 'echoTime' => 'Echo Time of the file', - 'AcquisitionProtocolID' => 'Scan type ID', - 'candID' => 'Candidate CandID', - 'sessionID' => 'Session ID', - 'visitLabel' => 'Visit Label', - 'echoNumber' => 'Echo Number of the scan', - 'seriesNumber' => 'Series Number of the scan', - 'imageType' => 'Image Type', - 'lorisScanType' => 'LORIS Scan Type name' + 'fileID' => 'FileID value', + 'file' => 'file path', + 'echoTime' => 'Echo Time of the file', + 'MriScanTypeID' => 'Scan type ID', + 'candID' => 'Candidate CandID', + 'sessionID' => 'Session ID', + 'visitLabel' => 'Visit Label', + 'echoNumber' => 'Echo Number of the scan', + 'seriesNumber' => 'Series Number of the scan', + 'imageType' => 'Image Type', + 'lorisScanType' => 'LORIS Scan Type name' } ... } @@ -427,7 +427,7 @@ sub getFileList { SELECT f.FileID, File, - AcquisitionProtocolID, + MriScanTypeID, EchoTime, c.CandID, s.Visit_label, @@ -436,11 +436,11 @@ sub getFileList { pf_echonb.Value as EchoNumber, pf_seriesnb.Value as SeriesNumber, pf_imagetype.Value as ImageType, - mst.Scan_type AS LorisScanType + mst.MriScanTypeName AS LorisScanType FROM files f JOIN session s ON (s.ID = f.SessionID) JOIN candidate c ON (c.CandID = s.CandID) -JOIN mri_scan_type mst ON (mst.ID = f.AcquisitionProtocolID) +JOIN mri_scan_type mst ON (mst.MriScanTypeID = f.MriScanTypeID) JOIN tarchive t ON (t.SessionID = s.ID) LEFT JOIN parameter_file pf_echonb ON (f.FileID=pf_echonb.FileID) AND pf_echonb.ParameterTypeID = ? LEFT JOIN parameter_file pf_seriesnb ON (f.FileID=pf_seriesnb.FileID) AND pf_seriesnb.ParameterTypeID = ? @@ -461,18 +461,18 @@ sub getFileList { my %file_list; my $i = 0; while ( my $rowhr = $st_handle->fetchrow_hashref()) { - $file_list{$i}{'fileID'} = $rowhr->{'FileID'}; - $file_list{$i}{'file'} = $rowhr->{'File'}; - $file_list{$i}{'AcquisitionProtocolID'} = $rowhr->{'AcquisitionProtocolID'}; - $file_list{$i}{'candID'} = $rowhr->{'CandID'}; - $file_list{$i}{'sessionID'} = $rowhr->{'SessionID'}; - $file_list{$i}{'visitLabel'} = $rowhr->{'Visit_label'}; - $file_list{$i}{'echoNumber'} = $rowhr->{'EchoNumber'}; - $file_list{$i}{'echoNumber'} =~ s/\.$//g if $file_list{$i}{'echoNumber'}; # remove trailing dot of the echo number - $file_list{$i}{'seriesNumber'} = $rowhr->{'SeriesNumber'}; - $file_list{$i}{'imageType'} = $rowhr->{'ImageType'}; - $file_list{$i}{'lorisScanType'} = $rowhr->{'LorisScanType'}; - $file_list{$i}{'AcqOrderPerModality'} = $rowhr->{'AcqOrderPerModality'}; + $file_list{$i}{'fileID'} = $rowhr->{'FileID'}; + $file_list{$i}{'file'} = $rowhr->{'File'}; + $file_list{$i}{'MriScanTypeID'} = $rowhr->{'MriScanTypeID'}; + $file_list{$i}{'candID'} = $rowhr->{'CandID'}; + $file_list{$i}{'sessionID'} = $rowhr->{'SessionID'}; + $file_list{$i}{'visitLabel'} = $rowhr->{'Visit_label'}; + $file_list{$i}{'echoNumber'} = $rowhr->{'EchoNumber'}; + $file_list{$i}{'echoNumber'} =~ s/\.$//g if $file_list{$i}{'echoNumber'}; # remove trailing dot of the echo number + $file_list{$i}{'seriesNumber'} = $rowhr->{'SeriesNumber'}; + $file_list{$i}{'imageType'} = $rowhr->{'ImageType'}; + $file_list{$i}{'lorisScanType'} = $rowhr->{'LorisScanType'}; + $file_list{$i}{'AcqOrderPerModality'} = $rowhr->{'AcqOrderPerModality'}; $i++; } @@ -601,7 +601,7 @@ sub makeNIIAndHeader { my $file_id = $file_list{$row}{'fileID'}; my $minc = $file_list{$row}{'file'}; my $minc_basename = basename($minc); - my $acq_protocol_id = $file_list{$row}{'AcquisitionProtocolID'}; + my $acq_protocol_id = $file_list{$row}{'MriScanTypeID'}; my $loris_scan_type = $file_list{$row}{'lorisScanType'}; my $session_id = $file_list{$row}{'sessionID'}; @@ -719,7 +719,7 @@ =head3 grep_bids_scan_categories_from_db($db_handle, $acq_protocol_id) 'BIDSCategoryName' => 'BIDS category to use for the NIfTI file, aka anat, func, fmap, dwi...', 'BIDSScanTypeSubCategory' => 'BIDS subcategory to use for the NIfTI file, aka task-rest, task-memory...', 'BIDSEchoNumber' => 'Echo Number associated with the NIfTI file', - 'Scan_type' => 'label of the LORIS Scan type from the mri_scan_type table' + 'ScanType' => 'label of the LORIS Scan type from the mri_scan_type table' } Note: BIDSEchoNumber and BIDSScanTypeSubCategory can be null for a given NIfTI file. @@ -738,15 +738,15 @@ sub grep_bids_scan_categories_from_db { bids_scan_type.BIDSScanType, bmstr.BIDSEchoNumber, bids_phase_encoding_direction.BIDSPhaseEncodingDirectionName, - mst.Scan_type + mst.MriScanTypeName AS ScanType FROM bids_mri_scan_type_rel bmstr - JOIN mri_scan_type mst ON mst.ID = bmstr.MRIScanTypeID + JOIN mri_scan_type mst ON mst.MriScanTypeID = bmstr.MRIScanTypeID JOIN bids_category USING (BIDSCategoryID) JOIN bids_scan_type USING (BIDSScanTypeID) LEFT JOIN bids_scan_type_subcategory USING (BIDSScanTypeSubCategoryID) LEFT JOIN bids_phase_encoding_direction USING (BIDSPhaseEncodingDirectionID) WHERE - mst.ID = ? + mst.MriScanTypeID = ? QUERY # Prepare and execute query my $st_handle = $db_handle->prepare($bids_query); @@ -1633,7 +1633,7 @@ sub grep_phasediff_associated_magnitude_files { my %magnitude_files; foreach my $row (keys %$loris_files_list) { - my $acq_prot_id = $loris_files_list->{$row}{'AcquisitionProtocolID'}; + my $acq_prot_id = $loris_files_list->{$row}{'MriScanTypeID'}; my $session_id = $loris_files_list->{$row}{'sessionID'}; my $echo_number = $loris_files_list->{$row}{'echoNumber'}; my $series_number = $loris_files_list->{$row}{'seriesNumber'}; @@ -1658,14 +1658,14 @@ sub grep_phasediff_associated_magnitude_files { =head3 grep_acquisitionProtocolID_from_BIDS_scan_type($db_handle, $bids_scan_type) -Greps the AcquisitionProtocolID associated to a BIDS magnitude file in the database. +Greps the MriScanTypeID associated to a BIDS magnitude file in the database. INPUTS: - $db_handle : database handle - $bids_scan_type: name of the BIDS scan type (for example: magnitude) OUTPUT: - - AcquisitionProtocolID associated to the BIDS scan type file in the database + - MriScanTypeID associated to the BIDS scan type file in the database =cut @@ -1674,9 +1674,9 @@ sub grep_acquisitionProtocolID_from_BIDS_scan_type { (my $scan_type_query = <{$row}{'file'}; - my $acq_prot_id = $magnitude_files_hash->{$row}{'AcquisitionProtocolID'}; + my $acq_prot_id = $magnitude_files_hash->{$row}{'MriScanTypeID'}; my $echo_nb = $magnitude_files_hash->{$row}{'echoNumber'}; my $file_id = $magnitude_files_hash->{$row}{'fileID'}; diff --git a/tools/run_defacing_script.pl b/tools/run_defacing_script.pl index 89cc3783c..7f2d1e5bb 100755 --- a/tools/run_defacing_script.pl +++ b/tools/run_defacing_script.pl @@ -296,12 +296,12 @@ =head1 METHODS =head3 grep_FileIDs_to_deface($session_id_ref, @modalities_to_deface) Queries the database for the list of acquisitions' FileID to be used to run the -defacing algorithm based on the provided list of SessionID and Scan_type to +defacing algorithm based on the provided list of SessionID and ScanType to restrict the search. INPUTS: - $session_id_ref : array of SessionIDs to use when grepping FileIDs - - @modalities_to_deface: array of Scan_type to use when grepping FileIDs + - @modalities_to_deface: array of ScanType to use when grepping FileIDs RETURNS: hash of matching FileIDs to be used to run the defacing algorithm organized in a hash as follows: @@ -333,9 +333,9 @@ sub grep_FileIDs_to_deface { } # base query - my $query = "SELECT SessionID, FileID, Scan_type, File " + my $query = "SELECT SessionID, FileID, mst.MriScanTypeName AS ScanType, File " . "FROM files f " - . "JOIN mri_scan_type mst ON (f.AcquisitionProtocolID = mst.ID) " + . "JOIN mri_scan_type mst ON (f.MriScanTypeID = mst.MriScanTypeID) " . "JOIN parameter_file pf USING (FileID) " . "JOIN parameter_type pt USING (ParameterTypeID) " . "WHERE pt.Name='acquisition:image_type' AND ( "; @@ -343,7 +343,7 @@ sub grep_FileIDs_to_deface { # add where clause for the different standard scan types to deface my @where; if (@modalities_to_deface) { - @where = map { "mst.Scan_type = ?" } @modalities_to_deface; + @where = map { "mst.MriScanTypeName = ?" } @modalities_to_deface; $query .= sprintf(" %s ", join(" OR ", @where)); } @@ -351,7 +351,7 @@ sub grep_FileIDs_to_deface { # where we will restrict the images to be defaced to the ones with a face based # on parameter_type 'acquisition:image_type' if (@special_cases) { - @where = map { "(mst.Scan_type = ? AND pf.Value LIKE ?)" } @special_cases; + @where = map { "(mst.MriScanTypeName = ? AND pf.Value LIKE ?)" } @special_cases; $query .= sprintf(" OR %s ", join(" OR ", @where)); } $query .= ")"; # closing the brackets of the modalities WHERE part of the query @@ -380,7 +380,7 @@ sub grep_FileIDs_to_deface { my %file_id_hash; while (my $row = $sth->fetchrow_hashref){ my $session_key = $row->{'SessionID'}; - my $scan_type_key = $row->{'Scan_type'}; + my $scan_type_key = $row->{'ScanType'}; my $file_id_value = $row->{'FileID'}; my $file_value = $row->{'File'}; $file_id_hash{$session_key}{$scan_type_key}{$file_id_value} = $file_value; @@ -439,10 +439,10 @@ sub check_if_deface_files_already_in_db { # base query my $query = "SELECT COUNT(*) " . " FROM files f " - . " JOIN mri_scan_type mst ON (mst.ID=f.AcquisitionProtocolID) "; + . " JOIN mri_scan_type mst USING (mst.MriScanTypeID=f.MriScanTypeID) "; # add where clause for the different defaced scan types - my @where = map { "mst.Scan_type = ?" } @defaced_scan_types; + my @where = map { "mst.MriScanTypeName = ?" } @defaced_scan_types; $query .= sprintf(" WHERE (%s) ", join(" OR ", @where)); $query .= " AND SessionID = ?"; @@ -481,7 +481,7 @@ sub grep_t1_ref_file { my %ref_file = ( "FileID" => $t1_fileIDs[0], "File" => $t1_files{$t1_fileIDs[0]}, - "Scan_type" => $ref_t1_scan_type + "ScanType" => $ref_t1_scan_type ); # remove that reference file from the hash of other files to deface @@ -523,7 +523,7 @@ sub determine_output_dir_and_basename { # determine the output base name for the *_deface_grid_0.mnc output my $output_basename = $output_basedir . basename($$ref_file{File}); - $output_basename =~ s/_$$ref_file{Scan_type}_\d\d\d\.mnc//i; + $output_basename =~ s/_$$ref_file{ScanType}_\d\d\d\.mnc//i; # return the output base directory and output basename return $output_basedir, $output_basename; @@ -620,7 +620,7 @@ sub fetch_defaced_files { my %defaced_images; # append the reference t1 defaced image to the hash $defaced_images{$deface_ref}{InputFileID} = $$ref_file{FileID}; - $defaced_images{$deface_ref}{Scan_type} = $$ref_file{Scan_type}; + $defaced_images{$deface_ref}{ScanType} = $$ref_file{ScanType}; # for each files in $session_files, append the defaced images to the hash foreach my $scan_type (keys %{ $session_files }) { @@ -629,7 +629,7 @@ sub fetch_defaced_files { my $deface_file = $deface_dir . '/' . basename($files{$fileID}); $deface_file =~ s/\.mnc$/_defaced\.mnc/; $defaced_images{$deface_file}{InputFileID} = $fileID; - $defaced_images{$deface_file}{Scan_type} = $scan_type; + $defaced_images{$deface_file}{ScanType} = $scan_type; } } @@ -665,7 +665,7 @@ sub register_defaced_files { foreach my $file (keys %{ $defaced_images }) { my $input_fileID = $$defaced_images{$file}{InputFileID}; - my $scan_type = $$defaced_images{$file}{Scan_type} . "-defaced"; + my $scan_type = $$defaced_images{$file}{ScanType} . "-defaced"; # verify that the scan type exists in mri_scan_type, if not create it create_defaced_scan_type($scan_type); @@ -701,13 +701,13 @@ =head3 create_defaced_scan_type($scan_type) sub create_defaced_scan_type { my ($scan_type) = @_; - my $select = "SELECT COUNT(*) FROM mri_scan_type WHERE Scan_type = ?"; + my $select = "SELECT COUNT(*) FROM mri_scan_type WHERE MriScanTypeName = ?"; my $select_sth = $dbh->prepare($select); $select_sth->execute($scan_type); my ($count) = $select_sth->fetchrow_array; unless ($count) { - my $insert = "INSERT INTO mri_scan_type SET Scan_type = ?"; + my $insert = "INSERT INTO mri_scan_type SET MriScanTypeName = ?"; my $sth = $dbh->prepare($insert); $sth->execute($scan_type); } diff --git a/uploadNeuroDB/NeuroDB/MRI.pm b/uploadNeuroDB/NeuroDB/MRI.pm index 0a5111512..ee0ef535e 100755 --- a/uploadNeuroDB/NeuroDB/MRI.pm +++ b/uploadNeuroDB/NeuroDB/MRI.pm @@ -106,8 +106,8 @@ RETURNS: the C or (if none exists) undef sub getScannerCandID { my ($scannerID, $db) = @_; - - my $mriScannerOB = + + my $mriScannerOB = NeuroDB::objectBroker::MriScannerOB->new(db => $db); my $resultRef = $mriScannerOB->get({ID => $scannerID}); return @$resultRef ? $resultRef->[0]->{'CandID'} : undef; @@ -124,10 +124,10 @@ exists, the method will try to create it using the supplied parameters. INPUTS: - $subjectIDref: hash reference of subject IDs - $studyDate : study date - - $dbh : database handle + - $dbh : database handle - $db : database object -RETURNS: an array of 2 elements: +RETURNS: an array of 2 elements: - A reference to a hash containing the session properties: C => session ID. C => project ID for the session. @@ -167,7 +167,7 @@ sub getSessionInformation { return (\%session, ''); } - + if (!$subjectIDref->{'createVisitLabel'}) { my $msg = sprintf( "Visit %s for candidate %d does not exist.", @@ -176,7 +176,7 @@ sub getSessionInformation { ); return (undef, $msg); } - + # Since we'll be creating a visit, ensure that the cohortID and ProjectID # have been defined in the profile file if (!defined $subjectIDref->{'CohortID'}) { @@ -206,11 +206,11 @@ sub getSessionInformation { ); return (undef, $msg); } - + # determine the visit number and centerID for the next session my $newVisitNo = 0; my $centerID = 0; - + if($subjectIDref->{'isPhantom'}) { # calibration data (PHANTOM_site_date | LIVING_PHANTOM_site_date | *test*) my @pscInfo = getPSC($subjectIDref->{'visitLabel'}, \$dbh, $db); @@ -245,11 +245,11 @@ sub getSessionInformation { } } } - + $newVisitNo = 1 unless $newVisitNo; $centerID = 0 unless $centerID; - # Insert the new session setting Current_stage to 'Not started' because that column is important + # Insert the new session setting Current_stage to 'Not started' because that column is important # to the behavioural data entry gui. $query = "INSERT INTO session " . "SET CandID = ?, " @@ -262,11 +262,11 @@ sub getSessionInformation { . " CohortID = ?, " . " ProjectID = ?"; $dbh->do( - $query, undef, - $subjectIDref->{'CandID'}, - $subjectIDref->{'visitLabel'}, - $centerID, - $newVisitNo, + $query, undef, + $subjectIDref->{'CandID'}, + $subjectIDref->{'visitLabel'}, + $centerID, + $newVisitNo, $subjectIDref->{'CohortID'}, $subjectIDref->{'ProjectID'} ); @@ -313,7 +313,7 @@ sub identify_scan_db { my $visitLabel = ${subjectref}->{'visitLabel'}; my $projectID = ${subjectref}->{'ProjectID'}; my $cohortID = ${subjectref}->{'CohortID'}; - + my $tarchiveID = $tarchiveInfoRef->{'TarchiveID'}; # get parameters from minc header @@ -365,10 +365,10 @@ sub identify_scan_db { Serial_number => $fileref->getParameter('device_serial_number'), Software => $fileref->getParameter('software_versions') }); - + # default ScannerID to NULL if we have no better clue. my $ScannerID = @$resultsRef> 0 ? $resultsRef->[0]->{'ID'} : undef; - + #===========================================================# # Get the list of lines in the mri_protocol table that # # apply to the given scan based on the center name # @@ -393,17 +393,17 @@ sub identify_scan_db { $query .= defined $visitLabel ? ' AND (mpgt.Visit_label IS NULL OR mpgt.Visit_label = ?)' : ' AND mpgt.Visit_label IS NULL'; - + $query .= ' ORDER BY CenterID ASC, ScannerID DESC'; my @bindValues = ($centerID, $ScannerID); push(@bindValues, $projectID) if defined $projectID; push(@bindValues, $cohortID) if defined $cohortID; push(@bindValues, $visitLabel) if defined $visitLabel; - + $sth = $${dbhr}->prepare($query); $sth->execute(@bindValues); - + my @rows = @{ $sth->fetchall_arrayref({}) }; # If no lines in the mri_protocol_group_target matches the ProjectID/CohortID/VisitLabel # then no lines of the mri_protocol table can be used to identify the scan type. This is most @@ -412,7 +412,7 @@ sub identify_scan_db { if(@rows == 0) { my $msg = "Warning! No protocol group can be used to determine the scan type " . "of $minc_location.\nIncorrect/incomplete setup of table mri_protocol_group_target: " - . " setting scan type to 'unknown'"; + . " setting scan type to 'unknown'"; print "$msg\n"; my $notify = NeuroDB::Notify->new( $dbhr ); $notify->spool('mri upload processing class', $msg, 0, 'MRI.pm', $uploadID, 'N', 'Y'); @@ -422,12 +422,12 @@ sub identify_scan_db { my %mriProtocolGroupIDs = map { $_->{'MriProtocolGroupID'} => 1 } @rows; if(keys %mriProtocolGroupIDs > 1) { my $msg = "Warning! More than one protocol group can be used to identify the scan type of $minc_location\n" - . "Ambiguous setup of table mri_protocol_group_target: setting scan type to 'unknown'"; + . "Ambiguous setup of table mri_protocol_group_target: setting scan type to 'unknown'"; print "$msg\n"; my $notify = NeuroDB::Notify->new( $dbhr ); $notify->spool('mri upload processing class', $msg, 0, 'MRI.pm', $uploadID, 'N', 'Y'); } else { - + $mriProtocolGroupID = [ keys %mriProtocolGroupIDs ]->[0]; # check against all possible scan types @@ -456,21 +456,10 @@ sub identify_scan_db { my $slice_thick_min = $rowref->{'slice_thickness_min'}; my $slice_thick_max = $rowref->{'slice_thickness_max'}; - - my $scan_type = &scan_type_id_to_text($rowref->{'Scan_type'}, $db); - # if no scan type found in mri_scan_type for $rowref->{'Scan_type'}, - # then throw an error - unless ($scan_type) { - NeuroDB::UnexpectedValueException->throw( - errorMessage => sprintf( - "Unknown acquisition protocol ID %d for ", - $rowref->{'Scan_type'} - ) - ); - } + my $scan_type = &scan_type_id_to_text($rowref->{'MriScanTypeID'}, $db); if(0) { - print "\tChecking ".$scan_type." ($rowref->{'Scan_type'}) ($series_description =~ $sd_regex)\n"; + print "\tChecking ".$scan_type." ($rowref->{'MriScanTypeID'}) ($series_description =~ $sd_regex)\n"; print "\t"; if($sd_regex && ($series_description =~ /$sd_regex/i)) { print "series_description\t"; @@ -650,9 +639,10 @@ sub scan_type_id_to_text { my $mriScanTypeOB = NeuroDB::objectBroker::MriScanTypeOB->new( db => $db ); + my $mriScanTypeRef = $mriScanTypeOB->get(0, { ID => $typeID }); - return @$mriScanTypeRef ? $mriScanTypeRef->[0]->{'Scan_type'} : undef; + return $mriScanTypeRef->[0]->{'MriScanTypeName'}; } =pod @@ -676,7 +666,7 @@ sub scan_type_text_to_id { db => $db ); my $mriScanTypeRef = $mriScanTypeOB->get( - 0, { Scan_type => $type } + 0, { MriScanTypeName => $type } ); return @$mriScanTypeRef ? $mriScanTypeRef->[0]->{'ID'} : undef; @@ -794,7 +784,7 @@ sub register_db { my $query = "INSERT INTO files SET "; my @field_array = ( 'File', 'SessionID', 'EchoTime', - 'CoordinateSpace', 'OutputType', 'AcquisitionProtocolID', + 'CoordinateSpace', 'OutputType', 'MriScanTypeID', 'FileType', 'InsertedByUserID', 'Caveat', 'SeriesUID', 'TarchiveSource', 'HrrtArchiveID', 'SourcePipeline', 'PipelineDate', 'SourceFileID', @@ -827,7 +817,7 @@ sub register_db { my $typeID = $file->getParameterTypeID($key); my $value = ''; $value = $dbh->quote(encode_utf8($${params{$key}})); - + if($query =~ /\)$/) { $query .= ",\n"; } $query .= "($fileID, $typeID, $value, UNIX_TIMESTAMP())"; @@ -992,10 +982,10 @@ sub findScannerID { Software => $softwareVersion, Serial_number => $serialNumber }); - + # Scanner exists return $resultsRef->[0]->{'ID'} if @$resultsRef; - + # register new scanners my $scanner_id = registerScanner($manufacturer, $model, $serialNumber, $softwareVersion, $centerID, $projectID, $dbhr, $db); @@ -1029,7 +1019,7 @@ sub registerScanner { # my $scanner_id = 0; my @results = (); my $dbh = $$dbhr; - + my $mriScannerOB = NeuroDB::objectBroker::MriScannerOB->new( db => $db ); my $resultsRef = $mriScannerOB->get( { Serial_number => $serialNumber } ); my $candID = @$resultsRef > 0 ? $resultsRef->[0]->{'CandID'} : undef; @@ -1043,8 +1033,8 @@ sub registerScanner { . "VALUES " . "($candID, 'scanner', $centerID, $projectID, NOW(), NOW(), 'NeuroDB::MRI', 'Scanner')"; $dbh->do($query); - } - + } + return $mriScannerOB->insertOne({ Manufacturer => $manufacturer, Model => $model, @@ -1116,9 +1106,9 @@ sub getPSC { ## Get the CenterID from the session table, if the PSCID and visit labels exist ## and could be extracted if ($PSCID && $visitLabel) { - my $query = "SELECT s.CenterID, p.MRI_alias FROM session s - JOIN psc p on p.CenterID=s.CenterID - JOIN candidate c on c.CandID=s.CandID + my $query = "SELECT s.CenterID, p.MRI_alias FROM session s + JOIN psc p on p.CenterID=s.CenterID + JOIN candidate c on c.CandID=s.CandID WHERE c.PSCID = ? AND s.Visit_label = ?"; my $sth = $${dbhr}->prepare($query); @@ -1129,13 +1119,13 @@ sub getPSC { } } - ## Otherwise, use the patient name to match it to the site alias or MRI alias + ## Otherwise, use the patient name to match it to the site alias or MRI alias my $pscOB = NeuroDB::objectBroker::PSCOB->new( db => $db ); my $pscsRef = $pscOB->get({ MRI_alias => { NOT => '' } }); foreach my $psc (@$pscsRef) { if ($patientName =~ /$psc->{'Alias'}/i || $patientName =~ /$psc->{'MRI_alias'}/i) { - return ($psc->{'MRI_alias'}, $psc->{'CenterID'}); + return ($psc->{'MRI_alias'}, $psc->{'CenterID'}); } } @@ -1384,8 +1374,8 @@ sub make_pics { my $sth = $${dbhr}->prepare("SELECT CandID, Visit_label FROM session WHERE ID=".$file->getFileDatum('SessionID')); $sth->execute(); my $rowhr = $sth->fetchrow_hashref(); - - my $acquisitionProtocol = scan_type_id_to_text($file->getFileDatum('AcquisitionProtocolID'), $db); + + my $acquisitionProtocol = scan_type_id_to_text($file->getFileDatum('MriScanTypeID'), $db); my $minc = $data_dir . '/' . $file->getFileDatum('File'); my $mincbase = basename($minc); $mincbase =~ s/\.mnc(\.gz)?$//; @@ -1838,11 +1828,11 @@ that could not be deleted. INPUTS: - @files: list of files to delete. - + =cut sub deleteFiles { my(@files) = @_; - + foreach(@files) { unlink $_ or warn "Warning! File '$_' could not be deleted: $!\n"; } diff --git a/uploadNeuroDB/NeuroDB/MRIProcessingUtility.pm b/uploadNeuroDB/NeuroDB/MRIProcessingUtility.pm index 255944db6..e3796440b 100755 --- a/uploadNeuroDB/NeuroDB/MRIProcessingUtility.pm +++ b/uploadNeuroDB/NeuroDB/MRIProcessingUtility.pm @@ -77,10 +77,10 @@ use DateTime; use Time::Piece; ## Define Constants ## -my $notify_detailed = 'Y'; # notification_spool message flag for messages to be displayed - # with DETAILED OPTION in the front-end/imaging_uploader -my $notify_notsummary = 'N'; # notification_spool message flag for messages to be displayed - # with SUMMARY Option in the front-end/imaging_uploader +my $notify_detailed = 'Y'; # notification_spool message flag for messages to be displayed + # with DETAILED OPTION in the front-end/imaging_uploader +my $notify_notsummary = 'N'; # notification_spool message flag for messages to be displayed + # with SUMMARY Option in the front-end/imaging_uploader =pod @@ -91,7 +91,7 @@ Creates a new instance of this class. The parameter C<$dbhr> is a reference to a C database handle, used to set the object's database handle, so that all the DB-driven methods will work. -INPUT: +INPUT: - $db : database object - $dbhr : DBI database handle reference - $debug : degug flag (1 for debug, 0 otherwise) @@ -112,7 +112,7 @@ sub new { "Usage: ".$params."->new(\$databaseHandleReference)" ); } - + unless(defined $db && blessed($db) && $db->isa('NeuroDB::Database')) { croak( "Usage: ".$params."->new(\$databaseObject)" @@ -155,7 +155,7 @@ sub new { $self->{'logfile'} = $logfile; $self->{'db'} = $db; $self->{'configOB'} = $configOB; - + return bless $self, $params; } @@ -355,7 +355,7 @@ sub extractAndParseTarchive { my $study_dir = $this->{TmpDir} . "/" . $this->extract_tarchive($tarchive, $upload_id, $seriesuid); my $ExtractSuffix = basename($tarchive, ".tar"); - # get rid of the tarchive Prefix + # get rid of the tarchive Prefix $ExtractSuffix =~ s/DCM_(\d{4}-\d{2}-\d{2})?_//; my $info = "head -n 12 $this->{TmpDir}/${ExtractSuffix}.meta"; my $header = `$info`; @@ -373,9 +373,9 @@ sub extractAndParseTarchive { This function does: 1) Determine subject's ID based on scanner ID and DICOM archive information. -2) Call the C function (will create the candidate if it does +2) Call the C function (will create the candidate if it does not exists and C config option is set to yes) -3) Call the C to validate the candidate information +3) Call the C to validate the candidate information (it will return a C if there is one) INPUTS: @@ -518,7 +518,7 @@ QUERY } return %tarchiveInfo; - + } else { my $tarchiveOB = NeuroDB::objectBroker::TarchiveOB->new(db => $this->{'db'}); @@ -635,7 +635,7 @@ sub determinePSC { $this->{LOG}->print($message); $this->spool($message, 'N', $upload_id, $notify_detailed); } - + return ($center_name, $centerID); } @@ -677,7 +677,7 @@ sub determineProjectID { =head3 determineScannerID($tarchiveInfo, $to_log, $centerID, $projectID, $upload_id) -Determines which scanner ID was used for DICOM acquisitions. Note, if +Determines which scanner ID was used for DICOM acquisitions. Note, if a scanner ID is not already associated to the scanner information found in the DICOM headers, then a new scanner will automatically be created. @@ -818,7 +818,7 @@ RETURNS: =cut sub getAcquisitionProtocol { - + my $this = shift; my ($file,$subjectIDsref,$tarchiveInfoRef,$centerID,$minc, $acquisitionProtocol,$bypass_extra_file_checks, $upload_id, $data_dir) = @_; @@ -837,8 +837,8 @@ sub getAcquisitionProtocol { $centerID, $subjectIDsref, $tarchiveInfoRef, - $file, - $this->{dbhr}, + $file, + $this->{dbhr}, $this->{'db'}, $minc, $upload_id, @@ -878,16 +878,16 @@ sub getAcquisitionProtocol { if ($bypass_extra_file_checks == 0) { $extra_validation_status = $this->extra_file_checks( - $acquisitionProtocolID, - $file, - $subjectIDsref, + $acquisitionProtocolID, + $file, + $subjectIDsref, $tarchiveInfoRef->{'PatientName'}, $data_dir ); $message = "\nextra_file_checks from table mri_protocol_check " . "logged in table mri_violations_log: $extra_validation_status\n"; $this->{LOG}->print($message); - + # 'warn' and 'exclude' are errors, while 'pass' is not # log in the notification_spool_table the $Verbose flag accordingly if ($extra_validation_status ne 'pass'){ @@ -902,7 +902,7 @@ sub getAcquisitionProtocol { $acquisitionProtocol, $this->{'db'} ); } - + return ($acquisitionProtocol, $acquisitionProtocolID, $extra_validation_status); } @@ -928,14 +928,14 @@ RETURNS: =cut sub extra_file_checks() { - + my $this = shift; my $scan_type = shift; my $file = shift; my $subjectIDsref = shift; my $pname = shift; my $data_dir = shift; - + my $candID = $subjectIDsref->{'CandID'}; my $projectID = $subjectIDsref->{'ProjectID'}; my $cohortID = $subjectIDsref->{'CohortID'}; @@ -944,7 +944,7 @@ sub extra_file_checks() { ## Step 1 - select all distinct exclude and warning headers for the scan type my $query = "SELECT DISTINCT(mpc.Header) FROM mri_protocol_checks mpc " . "JOIN mri_protocol_checks_group_target mpcgt USING(MriProtocolChecksGroupID) " - . "WHERE Scan_type=? AND Severity=?"; + . "WHERE MriScanTypeID=? AND Severity=?"; $query .= defined $projectID ? ' AND (mpcgt.ProjectID IS NULL OR mpcgt.ProjectID = ?)' : ' AND mpcgt.ProjectID IS NULL'; @@ -968,7 +968,7 @@ sub extra_file_checks() { push(@bindValues, $cohortID) if defined $cohortID; push(@bindValues, $visitLabel) if defined $visitLabel; $sth->execute(@bindValues); - + my @headers = map { $_->{'Header'} } @{ $sth->fetchall_arrayref({}) }; my %validFields = $this->loop_through_protocol_violations_checks( $scan_type, $severity, \@headers, $file, $projectID, $cohortID, $visitLabel @@ -1057,7 +1057,7 @@ sub loop_through_protocol_violations_checks { # query to fetch list of valid protocols in mri_protocol_checks table my $query = "SELECT * FROM mri_protocol_checks mpc " . "JOIN mri_protocol_checks_group_target mpcgt USING(MriProtocolChecksGroupID) " - . "WHERE mpc.Scan_type=? AND mpc.Severity=? AND mpc.Header=? "; + . "WHERE mpc.MriScanTypeID=? AND mpc.Severity=? AND mpc.Header=? "; $query .= defined $projectID ? ' AND (mpcgt.ProjectID IS NULL OR mpcgt.ProjectID = ?)' : ' AND mpcgt.ProjectID IS NULL'; @@ -1092,9 +1092,9 @@ sub loop_through_protocol_violations_checks { push(@valid_ranges, $valid_range); } push(@valid_regexs, $row->{'ValidRegex'}) if $row->{'ValidRegex'}; - + # the group on each row should be the same if the mri_protocol_checks_group_target - # table was setup properly + # table was setup properly $mriProtocolChecksGroupID = $row->{'MriProtocolChecksGroupID'}; } @@ -1155,7 +1155,7 @@ sub insert_into_mri_violations_log { 'PatientName' => $pname, 'CandID' => $candID, 'Visit_label' => $visit_label, - 'Scan_type' => $scan_type, + 'MriScanTypeID' => $scan_type, 'Severity' => $severity, 'EchoTime' => $file->getParameter('echo_time'), 'PhaseEncodingDirection' => $file->getParameter('phase_encoding_direction'), @@ -1192,7 +1192,7 @@ sub insert_into_mri_violations_log { && $dbViolLog->{'EchoTime'} eq $newViolationsLog{'EchoTime'} && $dbViolLog->{'EchoNumber'} eq $newViolationsLog{'EchoNumber'} && $dbViolLog->{'PhaseEncodingDirection'} eq $newViolationsLog{'PhaseEncodingDirection'} - && $dbViolLog->{'Scan_type'} eq $newViolationsLog{'Scan_type'} + && $dbViolLog->{'MriScanTypeID'} eq $newViolationsLog{'MriScanTypeID'} && $dbViolLog->{'Severity'} eq $newViolationsLog{'Severity'} && $dbViolLog->{'Header'} eq $newViolationsLog{'Header'} && $dbViolLog->{'Value'} eq $newViolationsLog{'Value'} @@ -1239,7 +1239,7 @@ sub loadAndCreateObjectFile { ########## load File object ################################ ############################################################ $message = "\n==> Loading file from disk $minc\n"; - $this->{LOG}->print($message); + $this->{LOG}->print($message); $this->spool($message, 'N', $upload_id, $notify_detailed); $file->loadFileFromDisk($minc); @@ -1274,7 +1274,7 @@ RETURNS: new name of the MINC file with path relative to C<$data_dir> =cut sub move_minc { - + my $this = shift; my ($minc, $subjectIDsref, $minc_type, $prefix, $data_dir, $hrrt, $upload_id) = @_; my ($new_name, $version,$cmd,$new_dir,$extension,@exts,$dir); @@ -1373,24 +1373,24 @@ sub registerScanIntoDB { || (defined(&Settings::isFileToBeRegisteredGivenProtocol) && Settings::isFileToBeRegisteredGivenProtocol($acquisitionProtocol) ) - ) + ) && (!defined($extra_validation_status) || $extra_validation_status !~ /exclude/)) { $${minc_file}->setFileData( - 'AcquisitionProtocolID', - $acquisitionProtocolID + 'MriScanTypeID', + $acquisitionProtocolID ); - - $message = "\nAcq protocol: $acquisitionProtocol " + + $message = "\nAcq protocol: $acquisitionProtocol " . "- ID: $acquisitionProtocolID\n"; $this->spool($message, 'N', $upload_id, $notify_detailed); ######################################################## - # set Date_taken = last modification timestamp ######### + # set Date_taken = last modification timestamp ######### # (can't seem to get creation timestamp) ################ ######################################################## $Date_taken = (stat($minc))[9]; - + ######################################################## ##### rename and move files ############################ ######################################################## @@ -1402,34 +1402,34 @@ sub registerScanIntoDB { ######################################################## #################### set the new file_path ############# - ######################################################## + ######################################################## $file_path = $minc; $file_path =~ s/$data_dir\///i; $${minc_file}->setFileData( - 'File', + 'File', $file_path ); #### set the acquisition_date my $study_start_date = ( - defined($${minc_file}->getParameter('study:start_year')) + defined($${minc_file}->getParameter('study:start_year')) && defined($${minc_file}->getParameter('study:start_month')) && defined($${minc_file}->getParameter('study:start_day')) - ? DateTime->new( - day => $${minc_file}->getParameter('study:start_day'), - month => $${minc_file}->getParameter('study:start_month'), - year => $${minc_file}->getParameter('study:start_year'), - )->strftime('%Y-%m-%d') + ? DateTime->new( + day => $${minc_file}->getParameter('study:start_day'), + month => $${minc_file}->getParameter('study:start_month'), + year => $${minc_file}->getParameter('study:start_year'), + )->strftime('%Y-%m-%d') : undef - ); + ); - my $acquisition_date = + my $acquisition_date = $${minc_file}->getParameter('acquisition_date') || $study_start_date || undef; print "Acquisition date: " . $acquisition_date . "\n"; - + $${minc_file}->setFileData( 'AcquisitionDate', $acquisition_date @@ -1450,7 +1450,7 @@ sub registerScanIntoDB { } ######################################################## - # register into the db fixme if I ever want a dry run ## + # register into the db fixme if I ever want a dry run ## ######################################################## $message = "\nRegistering file into database\n"; $this->spool($message, 'N', $upload_id, $notify_detailed); @@ -1532,7 +1532,7 @@ sub dicom_to_minc { my $pattern = join('$\'\t\'', quotemeta($studyUID), quotemeta($seriesUID), quotemeta($echo), quotemeta($seriesDesc)); # For each file in $study_dir... - my $d2m_cmd = "find $study_dir -type f "; + my $d2m_cmd = "find $study_dir -type f "; # ...invoke get_dicom_info to get its study UID, series instance UID, echo time, series description and file name $d2m_cmd .= "| $get_dicom_info -studyuid -attvalue 0020 000e -echo -series_descr -file -stdin"; @@ -1549,7 +1549,7 @@ sub dicom_to_minc { $d2m_cmd .= " | $converter -dname '' -stdin -clobber -usecoordinates $this->{TmpDir} "; } - # forcing bash to make sure ANSI-C quoting ($'\t') is supported. + # forcing bash to make sure ANSI-C quoting ($'\t') is supported. # It's not supported in some shells like Ubuntu dash. my $d2m_log = `/bin/bash -c "$d2m_cmd"`; @@ -1588,7 +1588,7 @@ INPUTS: =cut sub get_mincs { - + my $this = shift; my ($minc_files, $upload_id) = @_; my $message = ''; @@ -1629,7 +1629,7 @@ sub get_mincs { join("\n", @$minc_files)."\n"; $this->{LOG}->print($message); $this->spool($message, 'N', $upload_id, $notify_detailed); -} +} =pod @@ -1643,12 +1643,12 @@ INPUT: list of MINC files to concat =cut sub concat_mri { - + my $this = shift; my ($minc_files) = @_; my ($cmd,$log,$concat_count); ############################################################ - # make a list of the mincs to concat ####################### + # make a list of the mincs to concat ####################### # (avoid arg list too long errors) ######################### ############################################################ open CONCATFILES, ">$this->{TmpDir} /concatfilelist.txt"; @@ -1697,7 +1697,7 @@ sub registerProgs() { my @toregister = @_; foreach my $prog (@toregister) { my $present = `which $prog`; - if (!$present) { + if (!$present) { die("$prog not found") }; } @@ -1770,9 +1770,9 @@ sub moveAndUpdateTarchive { my $newArchiveLocationField = $newTarchiveLocation; $newArchiveLocationField =~ s/$tarchivePath\/?//g; $query = "UPDATE tarchive ". - " SET ArchiveLocation=" . + " SET ArchiveLocation=" . ${$this->{'dbhr'}}->quote($newArchiveLocationField) . - " WHERE DicomArchiveID=". + " WHERE DicomArchiveID=". ${$this->{'dbhr'}}->quote( $tarchiveInfo->{'DicomArchiveID'} ); @@ -1790,7 +1790,7 @@ Registers a new candidate in the C table. Note: before doing so, the following checks will be performed: 1) check that the C config option was set to yes -2) check that the C given in C<$subjectIDsref> is not already associated +2) check that the C given in C<$subjectIDsref> is not already associated to an existing candidate 3) check that the C given in C<$subjectIDsref> is not already associated to an existing candidate @@ -1820,15 +1820,15 @@ sub CreateMRICandidates { # If there already is a candidate with that PSCID, skip the creation. # Note that validateCandidate (which is called later on) will validate # that pscid and candid match so we don't do it here. - return if defined $pscID - && $pscID ne 'scanner' + return if defined $pscID + && $pscID ne 'scanner' && NeuroDB::MRI::subjectIDExists('PSCID', $pscID, $dbhr); - + # If there already is a candidate with that CandID, skip the creation. # Note that validateCandidate (which is called later on) will validate # that pscid and candid match so we don't do it here. return if NeuroDB::MRI::subjectIDExists('CandID', $candID, $dbhr); - + # return from the function if createCandidate config setting is not set my $configOB = $this->{'configOB'}; return if (!$configOB->getCreateCandidates()); @@ -2002,8 +2002,8 @@ sub which_directory { Check that the candidate's information derived from the patient name field of the DICOM files is valid (C and C of the candidate should -correspond to the same subject in the database). It will also check that the -Visit Label of C<$subjectIDsref> is a valid Visit Label present in the +correspond to the same subject in the database). It will also check that the +Visit Label of C<$subjectIDsref> is a valid Visit Label present in the C table. INPUT: subject's ID information hash ref @@ -2129,9 +2129,9 @@ sub computeSNR { my @modalities = $configOB->getComputeSnrModalities(); (my $query = <{debug}); @@ -2140,7 +2140,7 @@ QUERY while (my $row = $minc_file_arr->fetchrow_hashref()) { my $filename = $row->{'File'}; my $fileID = $row->{'FileID'}; - my $fileScanType = $row->{'Scan_type'}; + my $fileScanType = $row->{'ScanType'}; my $base = basename($filename); my $fullpath = "$data_dir/$filename"; my $message; @@ -2194,9 +2194,9 @@ sub orderModalitiesByAcq { my ($file, $acqProtID, $dataArr, $message, $sth); my ($tarchiveID, $upload_id)= @_; - my $queryAcqProt = "SELECT DISTINCT f.AcquisitionProtocolID ". + my $queryAcqProt = "SELECT DISTINCT f.MriScanTypeID ". "FROM files f ". - "WHERE f.TarchiveSource=?"; + "WHERE f.TarchiveSource=?"; if ($this->{debug}) { print $queryAcqProt . "\n"; @@ -2204,13 +2204,13 @@ sub orderModalitiesByAcq { my $acqArr = ${$this->{'dbhr'}}->prepare($queryAcqProt); $acqArr->execute($tarchiveID); - # For each of the files having this AcquisitionProtocolID - # load the file object to get the series_number + # For each of the files having this MriScanTypeID + # load the file object to get the series_number while (my $rowAcqProt = $acqArr->fetchrow_hashref()) { - $acqProtID = $rowAcqProt->{'AcquisitionProtocolID'}; + $acqProtID = $rowAcqProt->{'MriScanTypeID'}; my $queryDataArr = "SELECT f.FileID, f.AcqOrderPerModality ". "FROM files f ". - "WHERE f.TarchiveSource=? AND f.AcquisitionProtocolID=?"; + "WHERE f.TarchiveSource=? AND f.MriScanTypeID=?"; if ($this->{debug}) { print $queryDataArr . "\n"; diff --git a/uploadNeuroDB/NeuroDB/objectBroker/GetRole.pm b/uploadNeuroDB/NeuroDB/objectBroker/GetRole.pm index f4cfa3710..483195872 100644 --- a/uploadNeuroDB/NeuroDB/objectBroker/GetRole.pm +++ b/uploadNeuroDB/NeuroDB/objectBroker/GetRole.pm @@ -11,11 +11,11 @@ NeuroDB::objectBroker::GetRole -- A role for basic SELECT operations on the data =head1 DESCRIPTION This class provides methods used to accomplish basic C