Initial Python API changes for cmake adoption

- Move algorithm code from python/lib/mrtrix3/ to python/src/mrtrix3/.
- Modify python/bin/mrtrix3.py to reflect reduction of number of ways in which the Python API could potentially be found following calling mrtrix3.execute() in an executable command. This additionally includes the transition from the imp module to use importlib due to deprecation; this is a subset of #2735.
- For now, the capturing of the list of MRtrix3 executables in mrtrix3.EXE_LIST is disabled, as it is possible for MRtrix3 to be installed into a location that additionally includes other non-MRtrix3 executables. An alternative solution for this will need to be subsequently implemented.

python/bin/mrtrix3.py

@@ -14,53 +14,19 @@
 #
 # For more details, see http://www.mrtrix.org/.
 
-import imp, os, sys
-
-def imported(lib_path):
-  success = False
-  fp = None
-  try:
-    fp, pathname, description = imp.find_module('mrtrix3', [ lib_path ])
-    imp.load_module('mrtrix3', fp, pathname, description)
-    success = True
-  except ImportError:
-    pass
-  finally:
-    if fp:
-      fp.close()
-  return success
-
-# Can the MRtrix3 Python modules be found based on their relative location to this file?
-# Note that this includes the case where this file is a softlink within an external module,
-# which provides a direct link to the core installation
-if not imported (os.path.normpath (os.path.join ( \
-  os.path.dirname (os.path.realpath (__file__)), os.pardir, 'lib') )):
-
-  # If this file is a duplicate, which has been stored in an external module,
-  # we may be able to figure out the location of the core library using the
-  # build script.
-
-  # case 1: build is a symbolic link:
-  if not imported (os.path.join (os.path.dirname (os.path.realpath ( \
-      os.path.join (os.path.dirname(__file__), os.pardir, 'build'))), 'lib')):
-
-    # case 2: build is a file containing the path to the core build script:
-    try:
-      with open (os.path.join (os.path.dirname(__file__), os.pardir, 'build')) as fp:
-        for line in fp:
-          build_path = line.split ('#',1)[0].strip()
-          if build_path:
-            break
-    except IOError:
-      pass
-
-    if not imported (os.path.join (os.path.dirname (build_path), 'lib')):
-
-      sys.stderr.write('''
+import importlib, os, sys
+
+try:
+  spec = importlib.util.spec_from_file_location('mrtrix3', os.path.normpath(os.path.join(os.path.realpath(__file__), '..', 'lib', '__init__.py')))
+  module = importlib.util.module_from_spec(spec)
+  sys.modules[spec.name] = module
+  spec.loader.exec_module(module)
+except ImportError:
+  sys.stderr.write('''
 ERROR: Unable to locate MRtrix3 Python modules
 
 For detailed instructions, please refer to:
 https://mrtrix.readthedocs.io/en/latest/tips_and_tricks/external_modules.html
 ''')
-      sys.stderr.flush()
-      sys.exit(1)
+  sys.stderr.flush()
+  sys.exit(1)

python/lib/mrtrix3/__init__.py

@@ -39,7 +39,9 @@ class MRtrixError(MRtrixBaseError): #pylint: disable=unused-variable
 # Location of binaries that belong to the same MRtrix3 installation as the Python library being invoked
 BIN_PATH = os.path.abspath(os.path.join(os.path.abspath(os.path.dirname(os.path.abspath(__file__))), os.pardir, os.pardir, 'bin'))
 # Must remove the '.exe' from Windows binary executables
-EXE_LIST = [ os.path.splitext(name)[0] for name in os.listdir(BIN_PATH) ] #pylint: disable=unused-variable
+# TODO This needs to be read from a new file to be written by cmake --build;
+#   can't assume that binaries directory is not populated with other contents
+EXE_LIST = [ ] #pylint: disable=unused-variable
 
 
 # 'CONFIG' is a dictionary containing those entries present in the MRtrix config files

python/lib/mrtrix3/algorithm.py

@@ -23,10 +23,15 @@
 
 
 # Helper function for finding where the files representing different script algorithms will be stored
-# These will be in a sub-directory relative to this library file
+# These will be in a sub-directory relative to the executed file
 def _algorithms_path():
   from mrtrix3 import path #pylint: disable=import-outside-toplevel
-  return os.path.realpath(os.path.join(os.path.dirname(os.path.realpath(inspect.getouterframes(inspect.currentframe())[-1][1])), os.pardir, 'lib', 'mrtrix3', path.script_subdir_name()))
+  sys.stderr.write(os.path.realpath(inspect.getouterframes(inspect.currentframe())[-1][1]) + '\n')
+  return os.path.realpath(os.path.join(os.path.dirname(os.path.realpath(inspect.getouterframes(inspect.currentframe())[-1][1])),
+                                       os.pardir,
+                                       'src',
+                                       'mrtrix3',
+                                       path.script_srcdir_name()))
 
 
 
@@ -54,10 +59,13 @@ def usage(cmdline): #pylint: disable=unused-variable
   # Don't let Python 3 try to read incompatible .pyc files generated by Python 2 for no-longer-existent .py files
   pylist = get_list()
   base_parser = app.Parser(description='Base parser for construction of subparsers', parents=[cmdline])
-  subparsers = cmdline.add_subparsers(title='Algorithm choices', help='Select the algorithm to be used to complete the script operation; additional details and options become available once an algorithm is nominated. Options are: ' + ', '.join(get_list()), dest='algorithm')
+  subparsers = cmdline.add_subparsers(title='Algorithm choices',
+                                      help='Select the algorithm to be used to complete the script operation; '
+                                           'additional details and options become available once an algorithm is nominated. '
+                                           'Options are: ' + ', '.join(get_list()), dest='algorithm')
   for dummy_importer, package_name, dummy_ispkg in pkgutil.iter_modules( [ _algorithms_path() ] ):
     if package_name in pylist:
-      module = importlib.import_module(path.script_subdir_name() + '.' + package_name)
+      module = importlib.import_module(path.script_srcdir_name() + '.' + package_name)
       module.usage(base_parser, subparsers)
       initlist.extend(package_name)
   app.debug('Initialised algorithms: ' + str(initlist))
@@ -66,4 +74,4 @@ def usage(cmdline): #pylint: disable=unused-variable
 
 def get_module(name): #pylint: disable=unused-variable
   from mrtrix3 import path #pylint: disable=import-outside-toplevel
-  return sys.modules[path.script_subdir_name() + '.' + name]
+  return sys.modules[path.script_srcdir_name() + '.' + name]

python/lib/mrtrix3/path.py

@@ -17,7 +17,7 @@
 
 
 
-import ctypes, errno, inspect, os, random, shlex, shutil, string, subprocess, time
+import ctypes, errno, inspect, os, random, shlex, shutil, string, subprocess, sys, time
 from mrtrix3 import CONFIG
 
 
@@ -117,10 +117,10 @@ def name_temporary(suffix): #pylint: disable=unused-variable
 
 
 # Determine the name of a sub-directory containing additional data / source files for a script
-# This can be algorithm files in lib/mrtrix3/, or data files in share/mrtrix3/
+# This can be algorithm files in src/mrtrix3/, or data files in share/mrtrix3/
 # This function appears here rather than in the algorithm module as some scripts may
 #   need to access the shared data directory but not actually be using the algorithm module
-def script_subdir_name(): #pylint: disable=unused-variable
+def script_srcdir_name(): #pylint: disable=unused-variable
   from mrtrix3 import app #pylint: disable=import-outside-toplevel
   frameinfo = inspect.stack()[-1]
   try:

python/lib/mrtrix3/run.py

@@ -550,7 +550,7 @@ def exe_name(item):
 def version_match(item):
   from mrtrix3 import app #pylint: disable=import-outside-toplevel
   if not item in EXE_LIST:
-    app.debug('Command ' + item + ' not found in MRtrix3 bin/ directory')
+    app.debug('Command ' + item + ' not an MRtrix3 executable')
     return item
   exe_path_manual = os.path.join(BIN_PATH, exe_name(item))
   if os.path.isfile(exe_path_manual):

python/lib/mrtrix3/_5ttgen/freesurfer.py → python/src/mrtrix3/_5ttgen/freesurfer.py

@@ -56,7 +56,7 @@ def execute(): #pylint: disable=unused-variable
     lut_output_file_name = 'FreeSurfer2ACT_sgm_amyg_hipp.txt'
   else:
     lut_output_file_name = 'FreeSurfer2ACT.txt'
-  lut_output_path = os.path.join(path.shared_data_path(), path.script_subdir_name(), lut_output_file_name)
+  lut_output_path = os.path.join(path.shared_data_path(), path.script_srcdir_name(), lut_output_file_name)
   if not os.path.isfile(lut_output_path):
     raise MRtrixError('Could not find lookup table file for converting FreeSurfer parcellation output to tissues (expected location: ' + lut_output_path + ')')
 

python/lib/mrtrix3/_5ttgen/hsvs.py → python/src/mrtrix3/_5ttgen/hsvs.py

@@ -288,10 +288,10 @@ def execute(): #pylint: disable=unused-variable
       raise MRtrixError('FREESURFER_HOME environment variable not set; required for use of hippocampal subfields module')
     freesurfer_lut_file = os.path.join(os.environ['FREESURFER_HOME'], 'FreeSurferColorLUT.txt')
     check_file(freesurfer_lut_file)
-    hipp_lut_file = os.path.join(path.shared_data_path(), path.script_subdir_name(), 'hsvs', 'HippSubfields.txt')
+    hipp_lut_file = os.path.join(path.shared_data_path(), path.script_srcdir_name(), 'hsvs', 'HippSubfields.txt')
     check_file(hipp_lut_file)
     if hipp_subfield_has_amyg:
-      amyg_lut_file = os.path.join(path.shared_data_path(), path.script_subdir_name(), 'hsvs', 'AmygSubfields.txt')
+      amyg_lut_file = os.path.join(path.shared_data_path(), path.script_srcdir_name(), 'hsvs', 'AmygSubfields.txt')
       check_file(amyg_lut_file)
 
   if app.ARGS.sgm_amyg_hipp: