determine the toolchain generation in hooks

bin/submit_build.py

@@ -32,7 +32,7 @@
 from build_tools.clusters import ARCHS, PARTITIONS
 from build_tools.filetools import APPS_BRUSSEL, get_module
 from build_tools.lmodtools import submit_lmod_cache_job
-from build_tools.softinstall import mk_job_name, set_toolchain_generation, submit_build_job
+from build_tools.softinstall import mk_job_name, submit_build_job
 
 # repositories with easyconfigs
 VSCSOFTSTACK_ROOT = os.path.expanduser("~/vsc-software-stack")
@@ -79,15 +79,13 @@ def main():
         'sourcepath': '/apps/brussel/sources:/apps/gent/source',
         'installpath': os.path.join(APPS_BRUSSEL, os.getenv('VSC_OS_LOCAL'), LOCAL_ARCH),
         'buildpath': os.path.join(job['tmp'], 'eb-submit-build-fetch'),
-        'subdir-modules': 'modules',
         'hooks': hooks_hydra.__file__,
     }
 
     # Parse command line arguments
     options = {
         "arch": ("CPU architecture of the host system and the build", 'strlist', 'add', None, 'a'),
         "partition": ("Slurm partition for the build", 'strlist', 'add', None, 'P'),
-        "toolchain": ("Toolchain generation of the installation", None, "store", None, 't'),
         "extra-flags": ("Extra flags to pass to EasyBuild", None, "store", None, 'e'),
         "extra-sub-flags": ("Extra flags to pass to Slurm", None, "store", '', 'q'),
         "extra-mod-footer": ("Path to extra footer for module file", None, "store", None, 'f'),
@@ -173,14 +171,6 @@ def main():
     if opts.options.clang:
         job['langcode'] = 'C'
 
-    # Set target toolchain generation
-    job['tc_gen'] = set_toolchain_generation(easyconfig, user_toolchain=opts.options.toolchain)
-    if not job['tc_gen']:
-        logger.error("Unable to determine the toolchain generation, specify it with --toolchain")
-        sys.exit(1)
-
-    ebconf['subdir-modules'] = os.path.join('modules', job['tc_gen'])
-
     # Set robot paths
     if opts.options.pwd_robot_append:
         ebconf['robot-paths'] += ':' + os.getcwd()

src/build_tools/hooks_hydra.py

@@ -18,17 +18,18 @@
 """
 
 import os
+from pathlib import Path
 import time
 
 from flufl.lock import Lock, TimeOutError, NotLockedError
 
 from vsc.utils import fancylogger
 
 from easybuild.framework.easyconfig.constants import EASYCONFIG_CONSTANTS
-from easybuild.framework.easyconfig.easyconfig import letter_dir_for
+from easybuild.framework.easyconfig.easyconfig import letter_dir_for, get_toolchain_hierarchy
 from easybuild.tools import LooseVersion
 from easybuild.tools.build_log import EasyBuildError
-from easybuild.tools.config import source_paths
+from easybuild.tools.config import source_paths, ConfigurationVariables
 from easybuild.tools.filetools import mkdir
 from easybuild.tools.hooks import SANITYCHECK_STEP
 
@@ -64,6 +65,147 @@
 LOCAL_ARCH_SUFFIX = os.getenv('VSC_ARCH_SUFFIX')
 LOCAL_ARCH_FULL = f'{LOCAL_ARCH}{LOCAL_ARCH_SUFFIX}'
 
+VALID_TCGENS = ['2022a', '2023a']
+VALID_MODULES_SUBDIRS = VALID_TCGENS + ['system']
+VALID_TCS = ['foss', 'intel', 'gomkl', 'gimkl', 'gimpi']
+
+
+def get_tc_versions():
+    " build dict of (sub)toolchain-versions per valid generation "
+    tc_versions = {}
+    for toolcgen in VALID_TCGENS:
+        tc_versions[toolcgen] = []
+        for toolc in VALID_TCS:
+            try:
+                tc_versions[toolcgen].extend(get_toolchain_hierarchy({'name': toolc, 'version': toolcgen}))
+            except EasyBuildError:
+                # skip if no easyconfig found for toolchain-version
+                pass
+
+    return tc_versions
+
+
+def calc_tc_gen(name, version, tcname, tcversion, easyblock):
+    """
+    calculate the toolchain generation
+    return False if not valid
+    """
+    name_version = {'name': name, 'version': version}
+    toolchain = {'name': tcname, 'version': tcversion}
+    software = [name, version, tcname, tcversion, easyblock]
+
+    tc_versions = get_tc_versions()
+
+    # (software with) valid (sub)toolchain and version
+    for toolcgen in VALID_TCGENS:
+        if toolchain in tc_versions[toolcgen] or name_version in tc_versions[toolcgen]:
+            log_msg = f"Determined toolchain generation {toolcgen} for {software}"
+            return toolcgen, log_msg
+
+    # (software with) valid (sub)toolchain but invalid version
+    for toolcgen in VALID_TCGENS:
+        tcnames = [x['name'] for x in tc_versions[toolcgen]]
+        if toolchain['name'] in tcnames or name in tcnames:
+            log_msg = (f"Determined toolchain generation {toolcgen} for {software} is not valid."
+                       f" Choose one of {VALID_TCGENS}.")
+            return False, log_msg
+
+    # invalid toolchains
+    # all toolchains have 'system' toolchain, so we need to handle the invalid toolchains separately
+    # all toolchains have 'Toolchain' easyblock, so checking the easyblock is sufficient
+    if easyblock == 'Toolchain':
+        log_msg = f"Invalid toolchain {name} for {software}"
+        return False, log_msg
+
+    # software with 'system' toolchain: return 'system'
+    if tcname == 'system':
+        log_msg = f"Determined toolchain {tcname} for {software}"
+        return tcname, log_msg
+
+    log_msg = f"Invalid toolchain {tcname} for {software}"
+    return False, log_msg
+
+
+def update_module_install_paths(self):
+    " update module install paths unless subdir-modules uption is specified "
+
+    # default subdir_modules config var = 'modules'
+    # in hydra we change it to 'modules/<subdir>'
+    subdir_modules = Path(ConfigurationVariables()['subdir_modules']).parts
+
+    if len(subdir_modules) not in [1, 2] or subdir_modules[0] != 'modules':
+        log_msg = '[pre-fetch hook] Format of option subdir-modules %s is not valid. Must be modules/<subdir>.'
+        raise EasyBuildError(log_msg, os.path.join(*subdir_modules))
+
+    if len(subdir_modules) == 2:
+        subdir = subdir_modules[1]
+        if subdir not in VALID_MODULES_SUBDIRS:
+            log_msg = "[pre-fetch hook] Specified modules subdir %s is not valid. Choose one of %s."
+            raise EasyBuildError(log_msg, subdir, VALID_MODULES_SUBDIRS)
+        log_msg = "[pre-fetch hook] Option subdir-modules was set to %s, not updating module install paths."
+        self.log.info(log_msg, subdir_modules)
+        return
+
+    subdir, log_msg = calc_tc_gen(
+        self.name, self.version, self.toolchain.name, self.toolchain.version, self.cfg.easyblock)
+    if not subdir:
+        raise EasyBuildError("[pre-fetch hook] " + log_msg)
+    self.log.info("[pre-fetch hook] " + log_msg)
+
+    # insert subdir in module install path strings (normally between 'modules' and 'all')
+    installdir_mod = Path(self.installdir_mod).parts
+    self.installdir_mod = Path().joinpath(*installdir_mod[:-1], subdir, installdir_mod[-1]).as_posix()
+    self.log.info('[pre-fetch hook] Updated installdir_mod to %s.', self.installdir_mod)
+
+    mod_filepath = Path(self.mod_filepath).parts
+    self.mod_filepath = Path().joinpath(*mod_filepath[:-3], subdir, *mod_filepath[-3:]).as_posix()
+    self.log.info('[pre-fetch hook] Updated mod_filepath to %s.', self.mod_filepath)
+
+
+def acquire_fetch_lock(self):
+    " acquire fetch lock "
+    source_path = source_paths()[0]
+    full_source_path = os.path.join(source_path, letter_dir_for(self.name), self.name)
+    lock_name = full_source_path.replace('/', '_') + '.lock'
+
+    lock_dir = os.path.join(source_path, '.locks')
+    mkdir(lock_dir, parents=True)
+
+    wait_time = 0
+    wait_interval = 60
+    wait_limit = 3600
+
+    lock = Lock(os.path.join(lock_dir, lock_name), lifetime=wait_limit, default_timeout=1)
+    self.fetch_hook_lock = lock
+
+    while True:
+        try:
+            # try to acquire the lock
+            lock.lock()
+            self.log.info("[pre-fetch hook] Lock acquired: %s", lock.lockfile)
+            break
+
+        except TimeOutError as err:
+            if wait_time >= wait_limit:
+                error_msg = "[pre-fetch hook] Maximum wait time for lock %s to be released reached: %s sec >= %s sec"
+                raise EasyBuildError(error_msg, lock.lockfile, wait_time, wait_limit) from err
+
+            msg = "[pre-fetch hook] Lock %s held by another build, waiting %d seconds..."
+            self.log.debug(msg, lock.lockfile, wait_interval)
+            time.sleep(wait_interval)
+            wait_time += wait_interval
+
+
+def release_fetch_lock(self):
+    " release fetch lock "
+    lock = self.fetch_hook_lock
+    try:
+        lock.unlock()
+        self.log.info("[post-fetch hook] Lock released: %s", lock.lockfile)
+
+    except NotLockedError:
+        self.log.warning("[post-fetch hook] Could not release lock %s: was already released", lock.lockfile)
+
 
 def parse_hook(ec, *args, **kwargs):  # pylint: disable=unused-argument
     """Alter the parameters of easyconfigs"""
@@ -150,51 +292,13 @@ def parse_hook(ec, *args, **kwargs):  # pylint: disable=unused-argument
 
 def pre_fetch_hook(self):
     """Hook at pre-fetch level"""
-
-    # acquire fetch lock
-    source_path = source_paths()[0]
-    full_source_path = os.path.join(source_path, letter_dir_for(self.name), self.name)
-    lock_name = full_source_path.replace('/', '_') + '.lock'
-
-    lock_dir = os.path.join(source_path, '.locks')
-    mkdir(lock_dir, parents=True)
-
-    wait_time = 0
-    wait_interval = 60
-    wait_limit = 3600
-
-    lock = Lock(os.path.join(lock_dir, lock_name), lifetime=wait_limit, default_timeout=1)
-    self.fetch_hook_lock = lock
-
-    while True:
-        try:
-            # try to acquire the lock
-            lock.lock()
-            self.log.info("[pre-fetch hook] Lock acquired: %s", lock.lockfile)
-            break
-
-        except TimeOutError as err:
-            if wait_time >= wait_limit:
-                error_msg = "[pre-fetch hook] Maximum wait time for lock %s to be released reached: %s sec >= %s sec"
-                raise EasyBuildError(error_msg, lock.lockfile, wait_time, wait_limit) from err
-
-            msg = "[pre-fetch hook] Lock %s held by another build, waiting %d seconds..."
-            self.log.debug(msg, lock.lockfile, wait_interval)
-            time.sleep(wait_interval)
-            wait_time += wait_interval
+    update_module_install_paths(self)
+    acquire_fetch_lock(self)
 
 
 def post_fetch_hook(self):
     """Hook at post-fetch level"""
-
-    # release fetch lock
-    lock = self.fetch_hook_lock
-    try:
-        lock.unlock()
-        self.log.info("[post-fetch hook] Lock released: %s", lock.lockfile)
-
-    except NotLockedError:
-        self.log.warning("[post-fetch hook] Could not release lock %s: was already released", lock.lockfile)
+    release_fetch_lock(self)
 
 
 def pre_configure_hook(self, *args, **kwargs):  # pylint: disable=unused-argument

src/build_tools/jobtemplate.py

@@ -50,15 +50,9 @@
 mkdir -p ${eb_buildpath}
 
 # update MODULEPATH for cross-compilations
-if [ "${target_arch}" != "$$VSC_ARCH_LOCAL" ]; then
-    moddir="${eb_installpath}/modules"
-    # use modules from target arch and toolchain generation
-    CC_MODULEPATH=$${moddir}/${tc_gen}/all
-    # also add last 3 years of modules in case out-of-toolchain deps are needed
-    for modpath in $$(ls -1dr $${moddir}/*/all | head -n 6); do
-        CC_MODULEPATH="$$CC_MODULEPATH:$$modpath"
-    done
-    export MODULEPATH=$$CC_MODULEPATH
+local_arch="$$VSC_ARCH_LOCAL$$VSC_ARCH_SUFFIX"
+if [ "${target_arch}" != "$$local_arch" ]; then
+    export MODULEPATH=$${MODULEPATH//$$local_arch/${target_arch}}
 fi
 
 ${pre_eb_options} eb ${eb_options}

src/build_tools/package.py

@@ -16,7 +16,7 @@
 @author: Alex Domingo (Vrije Universiteit Brussel)
 """
 
-VERSION = '3.2.3'
+VERSION = '3.3.0'
 
 AUTHOR = {
     'wp': 'Ward Poelmans',

tests/conftest.py

@@ -19,6 +19,8 @@
 import os
 import pytest
 
+from easybuild.tools.options import set_up_configuration
+
 
 def pytest_addoption(parser):
     parser.addoption(
@@ -51,3 +53,9 @@ def realpath_apps_brussel(path):
 @pytest.fixture
 def mock_realpath_apps_brussel(monkeypatch):
     monkeypatch.setattr('os.path.realpath', realpath_apps_brussel)
+
+
+@pytest.fixture
+def set_up_config():
+    set_up_configuration(silent=True)
+    yield

tests/input/build_job_01.sh

@@ -28,15 +28,9 @@ mkdir -p $TMPDIR
 mkdir -p /tmp/eb-test-build
 
 # update MODULEPATH for cross-compilations
-if [ "skylake" != "$VSC_ARCH_LOCAL" ]; then
-    moddir="/apps/brussel/${VSC_OS_LOCAL}/skylake/modules"
-    # use modules from target arch and toolchain generation
-    CC_MODULEPATH=${moddir}/2019a/all
-    # also add last 3 years of modules in case out-of-toolchain deps are needed
-    for modpath in $(ls -1dr ${moddir}/*/all | head -n 6); do
-        CC_MODULEPATH="$CC_MODULEPATH:$modpath"
-    done
-    export MODULEPATH=$CC_MODULEPATH
+local_arch="$VSC_ARCH_LOCAL$VSC_ARCH_SUFFIX"
+if [ "skylake" != "$local_arch" ]; then
+    export MODULEPATH=${MODULEPATH//$local_arch/skylake}
 fi
 
  eb 

tests/input/build_job_02.sh

@@ -28,15 +28,9 @@ mkdir -p $TMPDIR
 mkdir -p /tmp/eb-test-build
 
 # update MODULEPATH for cross-compilations
-if [ "zen2" != "$VSC_ARCH_LOCAL" ]; then
-    moddir="/apps/brussel/${VSC_OS_LOCAL}/zen2-ib/modules"
-    # use modules from target arch and toolchain generation
-    CC_MODULEPATH=${moddir}/2020b/all
-    # also add last 3 years of modules in case out-of-toolchain deps are needed
-    for modpath in $(ls -1dr ${moddir}/*/all | head -n 6); do
-        CC_MODULEPATH="$CC_MODULEPATH:$modpath"
-    done
-    export MODULEPATH=$CC_MODULEPATH
+local_arch="$VSC_ARCH_LOCAL$VSC_ARCH_SUFFIX"
+if [ "zen2-ib" != "$local_arch" ]; then
+    export MODULEPATH=${MODULEPATH//$local_arch/zen2-ib}
 fi
 
 bwrap eb  --cuda-compute-capabilities=8.0