Python 2.7 and 3 support

PEP8 fixes
misc potential bug fixes

Author: brianv0

bin/scisql-deploy.py

@@ -12,14 +12,15 @@
 import sys
 import tempfile
 
+
 def parse_args():
 
     parser = argparse.ArgumentParser(
             description='''sciSQL deployment tool. Install sciSQL plugin in a
 MariaDB/MySQL running instance :\n
 - install shared library in MariaDB/MySQL plugin directory\n
-- install UDF in MariaDB/MySQL database''',
-)
+- install UDF in MariaDB/MySQL database'''
+    )
 
     # Defining option of each configuration step
     for step_name in configure.STEP_LIST:
@@ -34,56 +35,59 @@ def parse_args():
 
     # Logging management
     verbose_dict = {
-        'DEBUG'     : logging.DEBUG,
-        'INFO'      : logging.INFO,
-        'WARNING'   : logging.WARNING,
-        'ERROR'     : logging.ERROR,
-        'FATAL'     : logging.FATAL,
+        'DEBUG': logging.DEBUG,
+        'INFO': logging.INFO,
+        'WARNING': logging.WARNING,
+        'ERROR': logging.ERROR,
+        'FATAL': logging.FATAL,
     }
     verbose_arg_values = verbose_dict.keys()
-    parser.add_argument("-v", "--verbose-level", dest="verbose_str",
-            choices=verbose_arg_values,
-            default='INFO',
-            help="verbosity level (default: %(default)s)"
-            )
+    parser.add_argument("-v", "--verbose-level",
+                        dest="verbose_str",
+                        choices=verbose_arg_values,
+                        default='INFO',
+                        help="verbosity level (default: %(default)s)")
 
     # forcing options which may ask user confirmation
-    parser.add_argument("-f", "--force", dest="force", action='store_true',
-            default=False,
-            help="forcing removal of existing execution data (default: %(default)s)"
-            )
-
-    parser.add_argument("-T", "--tmp-dir", dest="tmp_dir",
-            default="/tmp",
-            help="""Path to directory where deployment temporary data will be stored."""
-            )
-
-    parser.add_argument("-m", "--mysql-dir", dest="mysql_dir",
-            default=os.getenv("MARIADB_DIR"),
-            help="""Path to the MariaDB/MySQL install directory.
-                    Default to MARIADB_DIR environement variable value if not empty (default: %(default)s)"""
-            )
-
-    parser.add_argument("-b", "--mysql-bin", dest="mysql_bin",
-            default="{mysql_dir}/bin/mysql",
-            help="""Path to the mysql command line client (e.g. /usr/local/bin/mysql).
-Used to CREATE and DROP the scisql UDFs after installation. (default: %(default)s)"""
-            )
-
-    parser.add_argument("-P", "--mysql-plugin-dir", dest="mysql_plugin_dir",
-            default="{mysql_dir}/lib/plugin",
-            help="full path to MariaDB/MySQL plugin directory (default: %(default)s)"
-            )
-
-    parser.add_argument("-S", "--mysql-socket", dest="mysql_socket",
-            default=None,
-            help="UNIX socket file for connecting to MySQL"
-            )
-
-    parser.add_argument("-U", "--mysql-user", dest="mysql_user",
-            default='root',
-            help="MySQL user name with admin privileges (default: %(default)s)"
-            )
+    parser.add_argument("-f", "--force",
+                        dest="force",
+                        action='store_true',
+                        default=False,
+                        help="forcing removal of existing execution data (default: %(default)s)")
+
+    parser.add_argument("-T", "--tmp-dir",
+                        dest="tmp_dir",
+                        default="/tmp",
+                        help="Path to directory where deployment temporary data will be stored.")
+
+    parser.add_argument("-m", "--mysql-dir",
+                        dest="mysql_dir",
+                        default=os.getenv("MARIADB_DIR"),
+                        help="""\
+Path to the MariaDB/MySQL install directory.
+Default to MARIADB_DIR environement variable value if not empty (default: %(default)s)""")
+
+    parser.add_argument("-b", "--mysql-bin",
+                        dest="mysql_bin",
+                        default="{mysql_dir}/bin/mysql",
+                        help="""\
+Path to the mysql command line client (e.g. /usr/local/bin/mysql).
+Used to CREATE and DROP the scisql UDFs after installation. (default: %(default)s)""")
+
+    parser.add_argument("-P", "--mysql-plugin-dir",
+                        dest="mysql_plugin_dir",
+                        default="{mysql_dir}/lib/plugin",
+                        help="full path to MariaDB/MySQL plugin directory (default: %(default)s)")
+
+    parser.add_argument("-S", "--mysql-socket",
+                        dest="mysql_socket",
+                        default=None,
+                        help="UNIX socket file for connecting to MySQL")
+
+    parser.add_argument("-U", "--mysql-user",
+                        dest="mysql_user",
+                        default='root',
+                        help="MySQL user name with admin privileges (default: %(default)s)")
 
     args = parser.parse_args()
 
@@ -100,6 +104,7 @@ def parse_args():
     args.verbose_level = verbose_dict[args.verbose_str]
     return args
 
+
 def check_global_args(args):
     # replace default values if needed
     args.mysql_bin = args.mysql_bin.format(mysql_dir=args.mysql_dir)
@@ -108,8 +113,7 @@ def check_global_args(args):
     logging.info('Checking for mysql command line client')
     if not os.path.isfile(args.mysql_bin) or not os.access(args.mysql_bin, os.X_OK):
         logging.fatal('{0} does not  identify an executable. Use --mysql-dir or --mysql options.'
-            .format(args.mysql_bin)
-        )
+                      .format(args.mysql_bin))
         sys.exit(1)
 
     logging.info('Checking for mysql socket')
@@ -120,11 +124,12 @@ def check_global_args(args):
         mode = os.stat(args.mysql_socket).st_mode
         is_socket = stat.S_ISSOCK(mode)
         if not is_socket:
-            logging.fatal('Invalid MySQL socket. Use --mysql-socket options.'
-                .format(args.mysql_socket)
+            logging.fatal(
+                'Invalid MySQL socket. Use --mysql-socket options.'.format(args.mysql_socket)
             )
             sys.exit(1)
 
+
 def main():
 
     scisql_dir = os.path.abspath(
@@ -136,9 +141,8 @@ def main():
     args = parse_args()
 
     logging.basicConfig(format='%(levelname)s: %(message)s', level=args.verbose_level)
-    logging.info("sciSQL deployment tool\n"+
-                 "============================"
-    )
+    logging.info("sciSQL deployment tool\n" +
+                 "============================")
 
     check_global_args(args)
 
@@ -159,44 +163,46 @@ def main():
     try:
         tmp_dir = tempfile.mkdtemp(suffix='-scisql', dir=args.tmp_dir)
 
-        scisql_template_dir=os.path.join(scisql_dir, "templates")
+        scisql_template_dir = os.path.join(scisql_dir, "templates")
         configure.apply_templates(scisql_template_dir, tmp_dir)
 
         if configure.DEPLOY in args.step_list:
 
             logging.info("Deploying sciSQL")
 
             logging.info('Checking for mysql version')
-            script=os.path.join(tmp_dir, "check_mysql_version.sh")
+            script = os.path.join(tmp_dir, "check_mysql_version.sh")
             utils.run_command([script])
 
             logging.info('Checking for mysql plugins directory')
             if not args.mysql_plugin_dir or not os.path.isdir(args.mysql_plugin_dir):
-                logging.fatal('Invalid/missing MySQL plugin directory. Use --mysql-dir or --mysql-plugin-dir options.')
+                logging.fatal(
+                    'Invalid/missing MySQL plugin directory. Use --mysql-dir or --mysql-plugin-dir options.'
+                )
                 sys.exit(1)
 
-            logging.info("Deploying sciSQL shared library in {0}"
-                .format(args.mysql_plugin_dir)
-            )
+            logging.info("Deploying sciSQL shared library in {0}".format(args.mysql_plugin_dir))
             scisql_libname = const.SCISQL_LIBNAME
             # check for existing shared library file of the same version
             scisql_plugin_lib = os.path.join(
                 args.mysql_plugin_dir,
                 scisql_libname
             )
             if os.path.isfile(scisql_plugin_lib):
-                logging.warn("Previous sciSQL library of the same version found in MySQL plugin directory ({0})"
-                    .format(scisql_plugin_lib) +
-                    ". sciSQL plugin (version: {0}) is certainly already deployed.".format(const.SCISQL_VERSION)
-                )
+                err_msg = """Previous sciSQL library of the same version found in MySQL plugin directory \
+({0}). sciSQL plugin (version: {1}) is certainly already deployed."""
+                logging.warning(err_msg.format(scisql_plugin_lib, const.SCISQL_VERSION))
                 if args.force or utils.user_yes_no_query(
                     "WARNING: Do you want to replace this shared library"
                     + " by current one and continue ? Answer 'yes' only if you know what you are doing."
                 ):
-                        logging.warn("Forcing sciSQL deployment over an existing sciSQL plugin of the same version.")
+                    logging.warning(
+                        "Forcing sciSQL deployment over an existing sciSQL plugin of the same version."
+                    )
                 else:
-                    logging.info("Stopping sciSQL deployment,"
-                        + "please remove previous sciSQL plugin from MySQL")
+                    logging.info(
+                        "Stopping sciSQL deployment, please remove previous sciSQL plugin from MySQL"
+                    )
                     sys.exit(1)
 
             scisql_lib = os.path.join(
@@ -211,24 +217,24 @@ def main():
 
             shutil.copy(scisql_lib, args.mysql_plugin_dir)
 
-            script=os.path.join(tmp_dir, configure.DEPLOY + ".sh")
+            script = os.path.join(tmp_dir, configure.DEPLOY + ".sh")
             utils.run_command([script])
 
         if configure.TEST in args.step_list:
-            script=os.path.join(tmp_dir, configure.TEST + ".sh")
+            script = os.path.join(tmp_dir, configure.TEST + ".sh")
             utils.run_command([script])
 
         if configure.UNDEPLOY in args.step_list:
-            script=os.path.join(tmp_dir, configure.UNDEPLOY + ".sh")
+            script = os.path.join(tmp_dir, configure.UNDEPLOY + ".sh")
             utils.run_command([script])
 
     finally:
         if logging.getLogger().getEffectiveLevel() > logging.DEBUG:
             shutil.rmtree(tmp_dir)
         else:
             logging.debug("Temporary directory {0} ".format(tmp_dir) +
-                "not removed in order to enable post-mortem analysis. " +
-                "Remove it manually.")
+                          "not removed in order to enable post-mortem analysis. " +
+                          "Remove it manually.")
 
 if __name__ == '__main__':
     main()

python/scisql/configure.py

@@ -1,8 +1,6 @@
 from scisql import const
-from distutils.util import strtobool
 import logging
 import os
-import subprocess
 import sys
 import string
 
@@ -11,40 +9,39 @@
 STEP_DOC = dict(
     zip(STEP_LIST,
         [
-        """Deploy sciSQL plugin in a MySQL running instance""",
-        """Launch tests on sciSQL plugin install """,
-        """Undeploy sciSQL plugin from a MySQL running instance""",
-        ]
-    )
+            "Deploy sciSQL plugin in a MySQL running instance",
+            "Launch tests on sciSQL plugin install ",
+            "Undeploy sciSQL plugin from a MySQL running instance",
+        ])
 )
 
 DEPLOY = STEP_LIST[0]
 TEST = STEP_LIST[1]
 UNDEPLOY = STEP_LIST[2]
+CONFIG = dict()
+
 
 def getConfig():
-    return config
+    return CONFIG
+
 
 def init_config(scisql_base, mysql_bin, mysql_user, mysql_password, mysql_socket):
 
-    global config
-    config = dict()
+    global CONFIG
     section = 'scisql'
-    config[section] = dict()
-    config[section]['base'] = scisql_base
-    config[section]['prefix'] = const.SCISQL_PREFIX
-    config[section]['version'] = const.SCISQL_VERSION
-    config[section]['vsuffix'] = const.SCISQL_VSUFFIX
-    config[section]['libname'] = const.SCISQL_LIBNAME
+    CONFIG[section] = dict()
+    CONFIG[section]['base'] = scisql_base
+    CONFIG[section]['prefix'] = const.SCISQL_PREFIX
+    CONFIG[section]['version'] = const.SCISQL_VERSION
+    CONFIG[section]['vsuffix'] = const.SCISQL_VSUFFIX
+    CONFIG[section]['libname'] = const.SCISQL_LIBNAME
     section = 'mysqld'
-    config[section] = dict()
-    config[section]['bin'] = mysql_bin
-    config[section]['user'] = mysql_user
-    config[section]['pass'] = mysql_password
-    config[section]['sock'] = mysql_socket
+    CONFIG[section] = dict()
+    CONFIG[section]['bin'] = mysql_bin
+    CONFIG[section]['user'] = mysql_user
+    CONFIG[section]['pass'] = mysql_password
+    CONFIG[section]['sock'] = mysql_socket
 
-def getConfig():
-    return config
 
 def _get_template_params():
     """ Compute templates parameters from Qserv meta-configuration file
@@ -54,29 +51,31 @@ def _get_template_params():
     config = getConfig()
 
     params_dict = {
-    'PATH': os.environ.get('PATH'),
-    'MYSQL_BIN': config['mysqld']['bin'],
-    'MYSQLD_SOCK': config['mysqld']['sock'],
-    'MYSQLD_USER': config['mysqld']['user'],
-    'MYSQLD_PASS': config['mysqld']['pass'],
-    'SCISQL_BASE': config['scisql']['base'],
-    'SCISQL_PREFIX': config['scisql']['prefix'],
-    'SCISQL_VERSION': config['scisql']['version'],
-    'SCISQL_VSUFFIX': config['scisql']['vsuffix'],
-    'SCISQL_LIBNAME': config['scisql']['libname'],
+        'PATH': os.environ.get('PATH'),
+        'MYSQL_BIN': config['mysqld']['bin'],
+        'MYSQLD_SOCK': config['mysqld']['sock'],
+        'MYSQLD_USER': config['mysqld']['user'],
+        'MYSQLD_PASS': config['mysqld']['pass'],
+        'SCISQL_BASE': config['scisql']['base'],
+        'SCISQL_PREFIX': config['scisql']['prefix'],
+        'SCISQL_VERSION': config['scisql']['version'],
+        'SCISQL_VSUFFIX': config['scisql']['vsuffix'],
+        'SCISQL_LIBNAME': config['scisql']['libname'],
     }
 
     logger.debug("Input parameters :\n {0}".format(params_dict))
 
     return params_dict
 
+
 def _set_perms(file):
     extension = os.path.splitext(file)[1]
     if extension in [".sh"]:
-        os.chmod(file, 0750)
+        os.chmod(file, 0o750)
     # all other files are configuration files
     else:
-        os.chmod(file, 0640)
+        os.chmod(file, 0o640)
+
 
 def apply_tpl(src_file, target_file):
     """ Creating one configuration file from one template
@@ -92,7 +91,7 @@ def apply_tpl(src_file, target_file):
     out_cfg = t.safe_substitute(**params_dict)
     for match in t.pattern.findall(t.template):
         name = match[1]
-        if len(name) != 0 and not params_dict.has_key(name):
+        if len(name) != 0 and name not in params_dict:
             logger.fatal("Template \"%s\" in file %s is not defined in configuration tool", name, src_file)
             sys.exit(1)
 
@@ -102,6 +101,7 @@ def apply_tpl(src_file, target_file):
     with open(target_file, "w") as cfg:
         cfg.write(out_cfg)
 
+
 def apply_templates(template_root, dest_root):
 
     logger = logging.getLogger()
@@ -123,6 +123,7 @@ def apply_templates(template_root, dest_root):
 
     return True
 
+
 class ScisqlConfigTemplate(string.Template):
     delimiter = '{{'
     pattern = r'''