Skip to content

Commit

Permalink
added support for pydra usage for python cmd with algorithm
Browse files Browse the repository at this point in the history
  • Loading branch information
@tclose
tclose committed Aug 2, 2023
1 parent c2b4b93 commit 2c9c938
Show file tree
Hide file tree
Showing 3 changed files with 123 additions and 75 deletions.
83 changes: 52 additions & 31 deletions lib/mrtrix3/algorithm.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,51 +19,72 @@


import importlib, inspect, os, pkgutil, sys

from pathlib import Path


# Helper function for finding where the files representing different script algorithms will be stored
# These will be in a sub-directory relative to this library file
def _algorithms_path():
from mrtrix3 import path #pylint: disable=import-outside-toplevel
return os.path.realpath(os.path.join(os.path.dirname(os.path.realpath(inspect.getouterframes(inspect.currentframe())[-1][1])), os.pardir, 'lib', 'mrtrix3', path.script_subdir_name()))

from mrtrix3 import path # pylint: disable=import-outside-toplevel
return str(Path(__file__).parent / "dwi2mask") # DEBUGGING, PLEASE REMOVE
return os.path.realpath(
os.path.join(
os.path.dirname(
os.path.realpath(inspect.getouterframes(inspect.currentframe())[-1][1])
),
os.pardir,
"lib",
"mrtrix3",
path.script_subdir_name(),
)
)


# This function needs to be safe to run in order to populate the help page; that is, no app initialisation has been run
def get_list(): #pylint: disable=unused-variable
from mrtrix3 import app #pylint: disable=import-outside-toplevel
algorithm_list = [ ]
for filename in os.listdir(_algorithms_path()):
filename = filename.split('.')
if len(filename) == 2 and filename[1] == 'py' and filename[0] != '__init__':
algorithm_list.append(filename[0])
algorithm_list = sorted(algorithm_list)
app.debug('Found algorithms: ' + str(algorithm_list))
return algorithm_list
def get_list(): # pylint: disable=unused-variable
from mrtrix3 import app # pylint: disable=import-outside-toplevel

algorithm_list = []
for filename in os.listdir(_algorithms_path()):
filename = filename.split(".")
if len(filename) == 2 and filename[1] == "py" and filename[0] != "__init__":
algorithm_list.append(filename[0])
algorithm_list = sorted(algorithm_list)
app.debug("Found algorithms: " + str(algorithm_list))
return algorithm_list


# Note: This function essentially duplicates the current state of app.cmdline in order for command-line
# options common to all algorithms of a particular script to be applicable once any particular sub-parser
# is invoked. Therefore this function must be called _after_ all such options are set up.
def usage(cmdline): #pylint: disable=unused-variable
from mrtrix3 import app, path #pylint: disable=import-outside-toplevel
sys.path.insert(0, os.path.realpath(os.path.join(_algorithms_path(), os.pardir)))
initlist = [ ]
# Don't let Python 3 try to read incompatible .pyc files generated by Python 2 for no-longer-existent .py files
pylist = get_list()
base_parser = app.Parser(description='Base parser for construction of subparsers', parents=[cmdline])
subparsers = cmdline.add_subparsers(title='Algorithm choices', help='Select the algorithm to be used to complete the script operation; additional details and options become available once an algorithm is nominated. Options are: ' + ', '.join(get_list()), dest='algorithm')
for dummy_importer, package_name, dummy_ispkg in pkgutil.iter_modules( [ _algorithms_path() ] ):
if package_name in pylist:
module = importlib.import_module(path.script_subdir_name() + '.' + package_name)
module.usage(base_parser, subparsers)
initlist.extend(package_name)
app.debug('Initialised algorithms: ' + str(initlist))
def usage(cmdline): # pylint: disable=unused-variable
from mrtrix3 import app, path # pylint: disable=import-outside-toplevel

sys.path.insert(0, os.path.realpath(os.path.join(_algorithms_path(), os.pardir)))
initlist = []
# Don't let Python 3 try to read incompatible .pyc files generated by Python 2 for no-longer-existent .py files
pylist = get_list()
base_parser = app.Parser(
description="Base parser for construction of subparsers", parents=[cmdline]
)
subparsers = cmdline.add_subparsers(
title="Algorithm choices",
help="Select the algorithm to be used to complete the script operation; additional details and options become available once an algorithm is nominated. Options are: "
+ ", ".join(get_list()),
dest="algorithm",
)
for dummy_importer, package_name, dummy_ispkg in pkgutil.iter_modules(
[_algorithms_path()]
):
if package_name in pylist:
module = importlib.import_module("dwi2mask." + package_name)
# module = importlib.import_module(path.script_subdir_name() + "." + package_name)
module.usage(base_parser, subparsers)
initlist.extend(package_name)
app.debug("Initialised algorithms: " + str(initlist))


def get_module(name): # pylint: disable=unused-variable
from mrtrix3 import path # pylint: disable=import-outside-toplevel

def get_module(name): #pylint: disable=unused-variable
from mrtrix3 import path #pylint: disable=import-outside-toplevel
return sys.modules[path.script_subdir_name() + '.' + name]
return sys.modules[path.script_subdir_name() + "." + name]
25 changes: 20 additions & 5 deletions lib/mrtrix3/app.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1091,6 +1091,19 @@ def print_group_options(group):

def print_usage_pydra(self):

if self._subparsers:

if len(sys.argv) == 3:
for alg in self._subparsers._group_actions[0].choices:
if alg == sys.argv[-2]:
self._subparsers._group_actions[0].choices[alg].print_usage_pydra()
return
self.error('Invalid subparser nominated: ' + sys.argv[-2])
assert len(sys.argv) == 2
sys.stdout.write(",".join(self._subparsers._group_actions[0].choices))
sys.stdout.flush()
return

def get_arg_metadata(arg):
metadata = {
"help_string": arg.help,
Expand Down Expand Up @@ -1172,6 +1185,8 @@ def parse_type(type_):
)
outputs_str = re.sub(r"'#([a-zA-Z0-9_\[\]]+)#'", r"\1", str(outputs))

task_name = self.prog.replace(" ", "_")

text = (
"import typing\n"
"from pathlib import Path # noqa: F401\n"
Expand All @@ -1184,10 +1199,10 @@ def parse_type(type_):
)

text += f"input_fields = {inputs_str}\n"
text += f"{self.prog}_input_spec = specs.SpecInfo(name='Input', fields=input_fields, bases=(specs.ShellSpec,))\n\n"
text += f"{task_name}_input_spec = specs.SpecInfo(name='Input', fields=input_fields, bases=(specs.ShellSpec,))\n\n"
text += f"output_fields = {outputs_str}\n"
text += f"{self.prog}_output_spec = specs.SpecInfo(name='Output', fields=output_fields, bases=(specs.ShellOutSpec,))\n\n"
text += f"class {self.prog}(ShellCommandTask):\n"
text += f"{task_name}_output_spec = specs.SpecInfo(name='Output', fields=output_fields, bases=(specs.ShellOutSpec,))\n\n"
text += f"class {task_name}(ShellCommandTask):\n"
indent = " "
text += indent + "\"\"\"\n"
text += indent + (self.description if self.description else "")
Expand All @@ -1204,8 +1219,8 @@ def parse_type(type_):
text += indent + '**Author:** ' + self._author + '\n\n'
text += indent + '**Copyright:** ' + self._copyright + '\n\n'
text += indent + "\"\"\"\n"
text += f" input_spec = {self.prog}_input_spec\n"
text += f" output_spec = {self.prog}_output_spec\n"
text += f" input_spec = {task_name}_input_spec\n"
text += f" output_spec = {task_name}_output_spec\n"
text += f" executable='{self.prog}'\n\n"

if HAVE_BLACK:
Expand Down
90 changes: 51 additions & 39 deletions pydra/pydra-auto-gen.py
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
import os
from pathlib import Path
import subprocess as sp
import typing as ty
import logging
import re
import click
import black.report
import black.parsing
Expand Down Expand Up @@ -45,46 +47,56 @@ def auto_gen_mrtrix3_pydra(cmd_dir: Path, output_dir: Path, log_errors: bool):
for cmd_name in sorted(os.listdir(cmd_dir)):
if cmd_name.startswith("_") or "." in cmd_name or cmd_name in IGNORE:
continue
try:
code_str = sp.check_output(
[str(cmd_dir / cmd_name), "__print_usage_pydra__"]
).decode("utf-8")
except sp.CalledProcessError:
if log_errors:
logger.error("Could not generate interface for '%s'", cmd_name)
continue
else:
raise

# Since Python identifiers can't start with numbers we need to rename 5tt*
# with fivetissuetype*
if cmd_name.startswith("5tt"):
old_name = cmd_name
cmd_name = old_name.replace("5tt", "fivetissuetype")
code_str = code_str.replace(
f"class {old_name}", f"class {cmd_name}"
)
code_str = code_str.replace(
f"{old_name}_input_spec", f"{cmd_name}_input_spec"
)
code_str = code_str.replace(
f"{old_name}_output_spec", f"{cmd_name}_input_spec"
)
try:
code_str = black.format_file_contents(
code_str, fast=False, mode=black.FileMode()
cmd = [str(cmd_dir / cmd_name)]
auto_gen_cmd(cmd, cmd_name, output_dir, log_errors)


def auto_gen_cmd(cmd: ty.List[str], cmd_name: str, output_dir: Path, log_errors: bool):

try:
code_str = sp.check_output(cmd + ["__print_usage_pydra__"]).decode("utf-8")
except sp.CalledProcessError:
if log_errors:
logger.error("Could not generate interface for '%s'", cmd_name)
return
else:
raise

if re.match(r"(\w+,)+\w+", code_str):
for algorithm in code_str.split(","):
auto_gen_cmd(
cmd + [algorithm], f"{cmd_name}_{algorithm}", output_dir, log_errors
)
except black.report.NothingChanged:
pass
except black.parsing.InvalidInput:
if log_errors:
logger.error("Could not parse generated interface for '%s'", cmd_name)
else:
raise
output_dir.mkdir(exist_ok=True)
with open(output_dir / (cmd_name + ".py"), "w") as f:
f.write(code_str)
logger.info("%s", cmd_name)

# Since Python identifiers can't start with numbers we need to rename 5tt*
# with fivetissuetype*
if cmd_name.startswith("5tt"):
old_name = cmd_name
cmd_name = old_name.replace("5tt", "fivetissuetype")
code_str = code_str.replace(
f"class {old_name}", f"class {cmd_name}"
)
code_str = code_str.replace(
f"{old_name}_input_spec", f"{cmd_name}_input_spec"
)
code_str = code_str.replace(
f"{old_name}_output_spec", f"{cmd_name}_input_spec"
)
try:
code_str = black.format_file_contents(
code_str, fast=False, mode=black.FileMode()
)
except black.report.NothingChanged:
pass
except black.parsing.InvalidInput:
if log_errors:
logger.error("Could not parse generated interface for '%s'", cmd_name)
else:
raise
output_dir.mkdir(exist_ok=True)
with open(output_dir / (cmd_name + ".py"), "w") as f:
f.write(code_str)
logger.info("%s", cmd_name)


if __name__ == "__main__":
Expand Down

0 comments on commit 2c9c938

Please sign in to comment.