Skip to content

Commit

Permalink
Merge pull request #106 from neuroscout/upload_only
Browse files Browse the repository at this point in the history 
Upload to Neurovault only command
  • Loading branch information
@adelavega
adelavega authored May 5, 2020
2 parents a187346 + b63381f commit 929d143
Show file tree
Hide file tree
Showing 5 changed files with 105 additions and 87 deletions.
6 changes: 4 additions & 2 deletions neuroscout_cli/cli.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
Usage:
neuroscout run [-dfuv -i <dir> -s <k> -w <dir> -c <n> -n <nv>] <outdir> <bundle_id>...
neuroscout install [-ui <dir>] <bundle_id>...
neuroscout upload [-f -n <nv>] <outdir> <bundle_id>...
neuroscout ls <bundle_id>
neuroscout -h | --help
neuroscout --version
Expand All @@ -25,6 +26,7 @@
Commands:
run Runs analysis.
install Installs a bundle and/or dataset.
upload Upload existing analysis results to Neurovault.
ls Lists the available files in a bundle's dataset.
Examples:
Expand Down Expand Up @@ -54,11 +56,11 @@ def main():
if hasattr(ncl, k) and val:
k = k[0].upper() + k[1:]
command = getattr(ncl, k)
if k in ['Run', 'Install']:
if k in ['Run', 'Install', 'Upload']:
bundles = args.pop('<bundle_id>')
# Loop over bundles
for bundle in bundles:
logging.info("Running analysis : {}".format(bundle))
logging.info("Analysis ID : {}".format(bundle))
args['<bundle_id>'] = bundle
command(deepcopy(args)).run()
sys.exit(0)
4 changes: 3 additions & 1 deletion neuroscout_cli/commands/__init__.py
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
from .install import Install
from .ls import Ls
from .run import Run
from .upload import Upload

__all__ = [
'Install',
'Ls',
'Run'
'Run',
'Upload'
]
7 changes: 5 additions & 2 deletions neuroscout_cli/commands/install.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -109,5 +109,8 @@ def _check_version(self):
)
sys.exit(1)

def run(self):
return self.download_data()
def run(self, download_data=True):
if download_data:
return self.download_data()
else:
return self.download_bundle()
167 changes: 85 additions & 82 deletions neuroscout_cli/commands/run.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,95 +15,98 @@
class Run(Command):
''' Command for running neuroscout workflows. '''

def run(self):
def run(self, upload_only=False):
# Download bundle and install dataset if necessary
install = Install(self.options.copy())
bundle_path = install.run()
preproc_path = str(install.preproc_dir.absolute())
bundle_path = install.run(download_data=(not upload_only))

out_dir = Path(self.options.pop('<outdir>')) / install.bundle_id
smoothing = self.options.pop('--smoothing')
model_path = (bundle_path / 'model.json').absolute()
fitlins_args = [
preproc_path,
str(out_dir),
'dataset',
f'--model={model_path}',
'--ignore=/(.*desc-confounds_regressors.tsv)/',
f'--derivatives={bundle_path} {preproc_path}',
f'--smoothing={smoothing}:Dataset'
]

verbose = self.options.pop('--verbose')
if verbose:
fitlins_args.append('-vvv')
work_dir = self.options.pop('--work-dir', None)
if work_dir:
work_dir = str(Path(work_dir).absolute() / self.bundle_id)
fitlins_args.append(f"--work-dir={work_dir}")

neurovault = self.options.pop('--neurovault', 'group')
nv_force = self.options.pop('--force-neurovault', False)
if neurovault not in ['disable', 'group', 'all']:
raise ValueError("Invalid neurovault option.")

# Fitlins invalid keys
for k in INVALID:
self.options.pop(k, None)

# Add remaining optional arguments
for name, value in self.options.items():
if name.startswith('--'):
if value is True:
fitlins_args.append(f'{name}')
elif value is not None and value is not False:
fitlins_args.append(f'{name}={value}')
else:
if value is not False and value is not None:
fitlins_args.append(f'{name} {value}')

# Call fitlins as if CLI
retcode = run_fitlins(fitlins_args)

if retcode == 0:
if neurovault != 'disable':

model = json.load(open(model_path, 'r'))
n_subjects = len(model['Input']['Subject'])

logging.info("Uploading results to NeuroVault...")

# Find files
images = out_dir / 'fitlins'

ses_dirs = [a for a in images.glob('ses*') if a.is_dir()]
if ses_dirs: # If session, look for stat files in session fld
images = images / ses_dirs[0]

group = [i for i in images.glob('task*statmap.nii.gz')
if re.match(
'.*stat-[t|F|variance|effect]+.*', i.name)]

if neurovault == 'all':
sub = [i for i in images.glob('sub*/*statmap.nii.gz')
if re.match('.*stat-[variance|effect]+.*', i.name)]
if not upload_only:
preproc_path = str(install.preproc_dir.absolute())
smoothing = self.options.pop('--smoothing')

fitlins_args = [
preproc_path,
str(out_dir),
'dataset',
f'--model={model_path}',
'--ignore=/(.*desc-confounds_regressors.tsv)/',
f'--derivatives={bundle_path} {preproc_path}',
f'--smoothing={smoothing}:Dataset'
]

verbose = self.options.pop('--verbose')
if verbose:
fitlins_args.append('-vvv')
work_dir = self.options.pop('--work-dir', None)
if work_dir:
work_dir = str(Path(work_dir).absolute() / self.bundle_id)
fitlins_args.append(f"--work-dir={work_dir}")

# Fitlins invalid keys
for k in INVALID:
self.options.pop(k, None)

# Add remaining optional arguments
for name, value in self.options.items():
if name.startswith('--'):
if value is True:
fitlins_args.append(f'{name}')
elif value is not None and value is not False:
fitlins_args.append(f'{name}={value}')
else:
sub = None

# Upload results NeuroVault
self.api.analyses.upload_neurovault(
id=self.bundle_id,
validation_hash=install.resources['validation_hash'],
group_paths=group, subject_paths=sub,
force=nv_force,
n_subjects=n_subjects)
else:
logging.error(
"\n"
"-----------------------------------------------------------\n"
"Model execution failed! \n"
f"neuroscout-cli version: {VERSION}\n"
"Update this program or revise your model, and try again.\n"
"If you believe there is a bug, please report it:\n"
"https://github.com/neuroscout/neuroscout-cli/issues\n"
"-----------------------------------------------------------\n"
)
if value is not False and value is not None:
fitlins_args.append(f'{name} {value}')

# Call fitlins as if CLI
retcode = run_fitlins(fitlins_args)

if retcode != 0:
logging.error(
"\n"
"-------------------------------------------------------\n"
"Model execution failed! \n"
f"neuroscout-cli version: {VERSION}\n"
"Update neuroscout-cli or revise your model, "
"and try again \n"
"If you believe there is a bug, please report it:\n"
"https://github.com/neuroscout/neuroscout-cli/issues\n"
"-------------------------------------------------------\n"
)

if neurovault != 'disable':
model = json.load(open(model_path, 'r'))
n_subjects = len(model['Input']['Subject'])

logging.info("Uploading results to NeuroVault...")

# Find files
images = out_dir / 'fitlins'

ses_dirs = [a for a in images.glob('ses*') if a.is_dir()]
if ses_dirs: # If session, look for stat files in session fld
images = images / ses_dirs[0]

group = [i for i in images.glob('task*statmap.nii.gz')
if re.match(
'.*stat-[t|F|variance|effect]+.*', i.name)]

if neurovault == 'all':
sub = [i for i in images.glob('sub*/*statmap.nii.gz')
if re.match('.*stat-[variance|effect]+.*', i.name)]
else:
sub = None

# Upload results NeuroVault
self.api.analyses.upload_neurovault(
id=self.bundle_id,
validation_hash=install.resources['validation_hash'],
group_paths=group, subject_paths=sub,
force=nv_force,
n_subjects=n_subjects)
8 changes: 8 additions & 0 deletions neuroscout_cli/commands/upload.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
from neuroscout_cli.commands.run import Run


class Upload(Run):
''' Command for running neuroscout workflows. '''

def run(self):
return super().run(upload_only=True)

0 comments on commit 929d143

Please sign in to comment.