Merge pull request #523 from xylar/simplify_local_mache

Enable easier mache testing

Author: xylar

conda/bootstrap.py

@@ -1,6 +1,4 @@
-#!/usr/bin/env python
-
-from __future__ import print_function
+#!/usr/bin/env python3
 
 import glob
 import grp
@@ -238,10 +236,10 @@ def get_env_setup(args, config, machine, compiler, mpi, env_type, source_path,
         activ_path, env_path, env_name, activate_env, spack_env
 
 
-def build_conda_env(env_type, recreate, machine, mpi, conda_mpi, version,
+def build_conda_env(env_type, recreate, mpi, conda_mpi, version,
                     python, source_path, conda_template_path, conda_base,
                     env_name, env_path, activate_base, use_local,
-                    local_conda_build, logger):
+                    local_conda_build, logger, local_mache):
 
     if env_type != 'dev':
         install_miniconda(conda_base, activate_base, logger)
@@ -284,7 +282,8 @@ def build_conda_env(env_type, recreate, machine, mpi, conda_mpi, version,
             conda_openmp = ''
         spec_file = template.render(supports_otps=supports_otps,
                                     mpi=conda_mpi, openmp=conda_openmp,
-                                    mpi_prefix=mpi_prefix)
+                                    mpi_prefix=mpi_prefix,
+                                    include_mache=not local_mache)
 
         spec_filename = f'spec-file-{conda_mpi}.txt'
         with open(spec_filename, 'w') as handle:
@@ -423,7 +422,7 @@ def build_spack_env(config, update_spack, machine, compiler, mpi, spack_env,
             f'scorpio@{scorpio}+pnetcdf~timing+internal-timing~tools+malloc')
 
     if albany != 'None':
-        specs.append(f'albany@{albany}+mpas+cxx17')
+        specs.append(f'albany@{albany}+mpas')
 
     yaml_template = f'{spack_template_path}/{machine}_{compiler}_{mpi}.yaml'
     if not os.path.exists(yaml_template):
@@ -802,6 +801,8 @@ def main():  # noqa: C901
 
     compass_version = get_version()
 
+    local_mache = args.mache_fork is not None and args.mache_branch is not None
+
     machine = None
     if not args.env_only:
         if args.machine is None:
@@ -889,10 +890,20 @@ def main():  # noqa: C901
 
         if previous_conda_env != conda_env_name:
             build_conda_env(
-                env_type, recreate, machine, mpi, conda_mpi, compass_version,
+                env_type, recreate, mpi, conda_mpi, compass_version,
                 python, source_path, conda_template_path, conda_base,
                 conda_env_name, conda_env_path, activate_base, args.use_local,
-                args.local_conda_build, logger)
+                args.local_conda_build, logger, local_mache)
+
+            if local_mache:
+                print('Install local mache\n')
+                commands = f'source {conda_base}/etc/profile.d/conda.sh && ' \
+                           f'source {conda_base}/etc/profile.d/mamba.sh && ' \
+                           f'conda activate {conda_env_name} && ' \
+                           'cd ../build_mache/mache && ' \
+                           'python -m pip install .'
+                check_call(commands, logger=logger)
+
             previous_conda_env = conda_env_name
 
             if env_type != 'dev':

conda/compass_env/spec-file.template

@@ -15,7 +15,9 @@ jigsaw=0.9.14
 jigsawpy=0.3.3
 jupyter
 lxml
+{% if include_mache %}
 mache=1.10.0
+{% endif %}
 matplotlib-base
 metis
 mpas_tools=0.17.0

conda/configure_compass_env.py

@@ -1,20 +1,8 @@
-#!/usr/bin/env python
-
-from __future__ import print_function
+#!/usr/bin/env python3
 
 import os
 import sys
-
-try:
-    from configparser import ConfigParser
-except ImportError:
-    import six
-    from six.moves import configparser
-
-    if six.PY2:
-        ConfigParser = configparser.SafeConfigParser
-    else:
-        ConfigParser = configparser.ConfigParser
+from configparser import ConfigParser
 
 from shared import (
     check_call,
@@ -41,36 +29,30 @@ def get_config(config_file):
 def bootstrap(activate_install_env, source_path, local_conda_build):
 
     print('Creating the compass conda environment\n')
-    bootstrap_command = '{}/conda/bootstrap.py'.format(source_path)
-    command = '{} && ' \
-              '{} {}'.format(activate_install_env, bootstrap_command,
-                             ' '.join(sys.argv[1:]))
+    bootstrap_command = f'{source_path}/conda/bootstrap.py'
+    command = f'{activate_install_env} && ' \
+              f'{bootstrap_command} {" ".join(sys.argv[1:])}'
     if local_conda_build is not None:
-        command = '{} --local_conda_build {}'.format(command,
-                                                     local_conda_build)
+        command = f'{command} --local_conda_build {local_conda_build}'
     check_call(command)
 
 
 def setup_install_env(env_name, activate_base, use_local, logger, recreate,
-                      conda_base):
+                      conda_base, mache):
     env_path = os.path.join(conda_base, 'envs', env_name)
     if use_local:
         channels = '--use-local'
     else:
         channels = ''
-    packages = 'progressbar2 jinja2 "mache=1.10.0"'
+    packages = f'progressbar2 jinja2 {mache}'
     if recreate or not os.path.exists(env_path):
         print('Setting up a conda environment for installing compass\n')
-        commands = '{} && ' \
-                   'mamba create -y -n {} {} {}'.format(activate_base,
-                                                        env_name, channels,
-                                                        packages)
+        commands = f'{activate_base} && ' \
+                   f'mamba create -y -n {env_name} {channels} {packages}'
     else:
         print('Updating conda environment for installing compass\n')
-        commands = '{} && ' \
-                   'mamba install -y -n {} {} {}'.format(activate_base,
-                                                         env_name, channels,
-                                                         packages)
+        commands = f'{activate_base} && ' \
+                   f'mamba install -y -n {env_name} {channels} {packages}'
 
     check_call(commands, logger=logger)
 
@@ -79,6 +61,12 @@ def main():
     args = parse_args(bootstrap=False)
     source_path = os.getcwd()
 
+    if args.tmpdir is not None:
+        try:
+            os.makedirs(args.tmpdir)
+        except FileExistsError:
+            pass
+
     config = get_config(args.config_file)
 
     conda_base = get_conda_base(args.conda_base, config, warn=True)
@@ -87,14 +75,14 @@ def main():
     env_name = 'compass_bootstrap'
 
     source_activation_scripts = \
-        'source {}/etc/profile.d/conda.sh && ' \
-        'source {}/etc/profile.d/mamba.sh'.format(conda_base, conda_base)
+        f'source {conda_base}/etc/profile.d/conda.sh && ' \
+        f'source {conda_base}/etc/profile.d/mamba.sh'
 
-    activate_base = '{} && mamba activate'.format(source_activation_scripts)
+    activate_base = f'{source_activation_scripts} && conda activate'
 
     activate_install_env = \
-        '{} && ' \
-        'mamba activate {}'.format(source_activation_scripts, env_name)
+        f'{source_activation_scripts} && ' \
+        f'conda activate {env_name}'
     try:
         os.makedirs('conda/logs')
     except OSError:
@@ -106,15 +94,34 @@ def main():
     # install miniconda if needed
     install_miniconda(conda_base, activate_base, logger)
 
+    local_mache = args.mache_fork is not None and args.mache_branch is not None
+    if local_mache:
+        mache = ''
+    else:
+        mache = '"mache=1.10.0"'
+
     setup_install_env(env_name, activate_base, args.use_local, logger,
-                      args.recreate, conda_base)
+                      args.recreate, conda_base, mache)
+
+    if local_mache:
+        print('Clone and install local mache\n')
+        commands = f'{activate_install_env} && ' \
+                   f'rm -rf conda/build_mache && ' \
+                   f'mkdir -p conda/build_mache && ' \
+                   f'cd conda/build_mache && ' \
+                   f'git clone -b {args.mache_branch} ' \
+                   f'[email protected]:{args.mache_fork}.git mache && ' \
+                   f'cd mache && ' \
+                   f'python -m pip install .'
+
+        check_call(commands, logger=logger)
 
     env_type = config.get('deploy', 'env_type')
     if env_type not in ['dev', 'test_release', 'release']:
-        raise ValueError('Unexpected env_type: {}'.format(env_type))
+        raise ValueError(f'Unexpected env_type: {env_type}')
 
     if env_type == 'test_release' and args.use_local:
-        local_conda_build = os.path.abspath('{}/conda-bld'.format(conda_base))
+        local_conda_build = os.path.abspath(f'{conda_base}/conda-bld')
     else:
         local_conda_build = None
 

conda/shared.py

@@ -1,17 +1,11 @@
-from __future__ import print_function
-
 import argparse
 import logging
 import os
 import platform
 import shutil
 import subprocess
 import sys
-
-try:
-    from urllib.request import Request, urlopen
-except ImportError:
-    from urllib2 import Request, urlopen
+from urllib.request import Request, urlopen
 
 
 def parse_args(bootstrap):
@@ -47,6 +41,10 @@ def parse_args(bootstrap):
                              "packages")
     parser.add_argument("--use_local", dest="use_local", action='store_true',
                         help="Use locally built conda packages (for testing).")
+    parser.add_argument("--mache_fork", dest="mache_fork",
+                        help="Point to a mache fork (and branch) for testing")
+    parser.add_argument("--mache_branch", dest="mache_branch",
+                        help="Point to a mache branch (and fork) for testing")
     parser.add_argument("--update_spack", dest="update_spack",
                         action='store_true',
                         help="If the shared spack environment should be "
@@ -78,6 +76,10 @@ def parse_args(bootstrap):
 
     args = parser.parse_args(sys.argv[1:])
 
+    if (args.mache_fork is None) != (args.mache_branch is None):
+        raise ValueError('You must supply both or neither of '
+                         '--mache_fork and --mache_branch')
+
     return args
 
 
@@ -92,9 +94,9 @@ def get_conda_base(conda_base, config, shared=False, warn=False):
             conda_base = os.path.abspath(
                 os.path.join(conda_exe, '..', '..'))
             if warn:
-                print('\nWarning: --conda path not supplied.  Using conda '
-                      'installed at:\n'
-                      '   {}\n'.format(conda_base))
+                print(f'\nWarning: --conda path not supplied.  Using conda '
+                      f'installed at:\n'
+                      f'   {conda_base}\n')
         else:
             raise ValueError('No conda base provided with --conda and '
                              'none could be inferred.')
@@ -118,9 +120,9 @@ def get_spack_base(spack_base, config):
 def check_call(commands, env=None, logger=None):
     print_command = '\n   '.join(commands.split(' && '))
     if logger is None:
-        print('\n Running:\n   {}\n'.format(print_command))
+        print(f'\n Running:\n   {print_command}\n')
     else:
-        logger.info('\nrunning:\n   {}\n'.format(print_command))
+        logger.info(f'\nrunning:\n   {print_command}\n')
 
     if logger is None:
         process = subprocess.Popen(commands, env=env, executable='/bin/bash',
@@ -154,8 +156,8 @@ def install_miniconda(conda_base, activate_base, logger):
             system = 'MacOSX'
         else:
             system = 'Linux'
-        miniconda = 'Mambaforge-{}-x86_64.sh'.format(system)
-        url = 'https://github.com/conda-forge/miniforge/releases/latest/download/{}'.format(miniconda)  # noqa: E501
+        miniconda = f'Mambaforge-{system}-x86_64.sh'
+        url = f'https://github.com/conda-forge/miniforge/releases/latest/download/{miniconda}'  # noqa: E501
         print(url)
         req = Request(url, headers={'User-Agent': 'Mozilla/5.0'})
         f = urlopen(req)
@@ -164,31 +166,30 @@ def install_miniconda(conda_base, activate_base, logger):
             outfile.write(html)
         f.close()
 
-        command = '/bin/bash {} -b -p {}'.format(miniconda, conda_base)
+        command = f'/bin/bash {miniconda} -b -p {conda_base}'
         check_call(command, logger=logger)
         os.remove(miniconda)
 
     backup_bashrc()
 
     print('Doing initial setup\n')
 
-    commands = '{} && ' \
-               'conda config --add channels conda-forge && ' \
-               'conda config --set channel_priority strict' \
-               ''.format(activate_base)
+    commands = f'{activate_base} && ' \
+               f'conda config --add channels conda-forge && ' \
+               f'conda config --set channel_priority strict'
 
     check_call(commands, logger=logger)
 
-    commands = '{} && ' \
-               'conda remove -y boa'.format(activate_base)
+    commands = f'{activate_base} && ' \
+               f'conda remove -y boa'
     try:
         check_call(commands, logger=logger)
     except subprocess.CalledProcessError:
         pass
 
-    commands = '{} && ' \
-               'mamba update -y --all && ' \
-               'mamba init'.format(activate_base)
+    commands = f'{activate_base} && ' \
+               f'mamba update -y --all && ' \
+               f'mamba init'
 
     check_call(commands, logger=logger)
 
@@ -200,7 +201,7 @@ def backup_bashrc():
     files = ['.bashrc', '.bash_profile']
     for filename in files:
         src = os.path.join(home_dir, filename)
-        dst = os.path.join(home_dir, '{}.conda_bak'.format(filename))
+        dst = os.path.join(home_dir, f'{filename}.conda_bak')
         if os.path.exists(src):
             shutil.copyfile(src, dst)
 
@@ -209,14 +210,14 @@ def restore_bashrc():
     home_dir = os.path.expanduser('~')
     files = ['.bashrc', '.bash_profile']
     for filename in files:
-        src = os.path.join(home_dir, '{}.conda_bak'.format(filename))
+        src = os.path.join(home_dir, f'{filename}.conda_bak')
         dst = os.path.join(home_dir, filename)
         if os.path.exists(src):
             shutil.move(src, dst)
 
 
 def get_logger(name, log_filename):
-    print('Logging to: {}\n'.format(log_filename))
+    print(f'Logging to: {log_filename}\n')
     try:
         os.remove(log_filename)
     except OSError: