From 48cb7aa608a7a1ef5304fb510361b70832410f7a Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Tue, 12 Nov 2024 17:25:34 +0100 Subject: [PATCH] Allow partition/environment extras to be used also as feature constraints --- docs/config_reference.rst | 4 +++- reframe/core/pipeline.py | 11 +++++++++-- reframe/core/runtime.py | 11 +++++++---- unittests/test_pipeline.py | 39 ++++++++++++++++++++++++++++++++------ 4 files changed, 52 insertions(+), 13 deletions(-) diff --git a/docs/config_reference.rst b/docs/config_reference.rst index 04d6959d60..cd7d8e510a 100644 --- a/docs/config_reference.rst +++ b/docs/config_reference.rst @@ -601,7 +601,7 @@ System Partition Configuration :required: No :default: ``[]`` - A list of job scheduler `resource specification <#custom-job-scheduler-resources>`__ objects. + A list of job scheduler :ref:`resource specification ` objects. .. py:attribute:: systems.partitions.processor @@ -747,6 +747,8 @@ ReFrame can launch containerized applications, but you need to configure properl If specified in conjunction with :attr:`~systems.partitions.container_platforms.env_vars`, it will be ignored. +.. _scheduler-resources: + Custom Job Scheduler Resources ============================== diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py index 8c3ab5e46c..5663f6e111 100644 --- a/reframe/core/pipeline.py +++ b/reframe/core/pipeline.py @@ -261,11 +261,18 @@ def pipeline_hooks(cls): #: of the :attr:`valid_systems` list, in which case an AND operation on #: these constraints is implied. For example, the test defining the #: following will be valid for all systems that have define both ``feat1`` - #: and ``feat2`` and set ``foo=1`` + #: and ``feat2`` and set ``foo=1``: #: #: .. code-block:: python #: - #: valid_systems = ['+feat1 +feat2 %foo=1'] + #: valid_systems = [r'+feat1 +feat2 %foo=1'] + #: + #: Any partition/environment extra or + #: :ref:`partition resource ` can be specified as a + #: feature constraint without having to explicitly state this in the + #: partition's/environment's feature list. For example, if ``key1`` is part + #: of the partition/environment extras list, then ``+key1`` will select + #: that partition or environment. #: #: For key/value pairs comparisons, ReFrame will automatically convert the #: value in the key/value spec to the type of the value of the diff --git a/reframe/core/runtime.py b/reframe/core/runtime.py index 24d754b991..e3fbce980f 100644 --- a/reframe/core/runtime.py +++ b/reframe/core/runtime.py @@ -311,11 +311,13 @@ def _is_valid_part(part, valid_systems): props[key] = val have_plus_feats = all( - ft in part.features or ft in part.resources + (ft in part.features or + ft in part.resources or ft in part.extras) for ft in plus_feats ) have_minus_feats = any( - ft in part.features or ft in part.resources + (ft in part.features or + ft in part.resources or ft in part.extras) for ft in minus_feats ) try: @@ -357,8 +359,9 @@ def _is_valid_env(env, valid_prog_environs): key, val = subspec[1:].split('=') props[key] = val - have_plus_feats = all(ft in env.features for ft in plus_feats) - have_minus_feats = any(ft in env.features + have_plus_feats = all(ft in env.features or ft in env.extras + for ft in plus_feats) + have_minus_feats = any(ft in env.features or ft in env.extras for ft in minus_feats) try: have_props = True diff --git a/unittests/test_pipeline.py b/unittests/test_pipeline.py index e5010e255c..6807088b91 100644 --- a/unittests/test_pipeline.py +++ b/unittests/test_pipeline.py @@ -556,7 +556,7 @@ def _assert_supported(valid_systems, valid_prog_environs, # Check AND in features and extras _assert_supported( - valid_systems=['+cuda +mpi %gpu_arch=v100'], + valid_systems=[r'+cuda +mpi %gpu_arch=v100'], valid_prog_environs=['*'], expected={} ) @@ -566,9 +566,18 @@ def _assert_supported(valid_systems, valid_prog_environs, expected={} ) - # Check OR in features ad extras + # Check OR in features and extras _assert_supported( - valid_systems=['+cuda +mpi', '%gpu_arch=v100'], + valid_systems=['+cuda +mpi', r'%gpu_arch=v100'], + valid_prog_environs=['*'], + expected={ + 'testsys:gpu': ['PrgEnv-gnu', 'builtin'] + } + ) + + # Check that extra keys can used as features + _assert_supported( + valid_systems=['+cuda +mpi', '+gpu_arch'], valid_prog_environs=['*'], expected={ 'testsys:gpu': ['PrgEnv-gnu', 'builtin'] @@ -640,7 +649,7 @@ def _assert_supported(valid_systems, valid_prog_environs, ) _assert_supported( valid_systems=['*'], - valid_prog_environs=['%bar=x'], + valid_prog_environs=[r'%bar=x'], expected={ 'testsys:gpu': [], 'testsys:login': ['PrgEnv-gnu'] @@ -648,7 +657,7 @@ def _assert_supported(valid_systems, valid_prog_environs, ) _assert_supported( valid_systems=['*'], - valid_prog_environs=['%foo=2'], + valid_prog_environs=[r'%foo=2'], expected={ 'testsys:gpu': ['PrgEnv-gnu'], 'testsys:login': [] @@ -656,7 +665,7 @@ def _assert_supported(valid_systems, valid_prog_environs, ) _assert_supported( valid_systems=['*'], - valid_prog_environs=['%foo=bar'], + valid_prog_environs=[r'%foo=bar'], expected={ 'testsys:gpu': [], 'testsys:login': [] @@ -671,6 +680,24 @@ def _assert_supported(valid_systems, valid_prog_environs, } ) + # Check that extra keys can used as features + _assert_supported( + valid_systems=['*'], + valid_prog_environs=['+foo +bar'], + expected={ + 'testsys:gpu': ['PrgEnv-gnu'], + 'testsys:login': ['PrgEnv-gnu'] + } + ) + _assert_supported( + valid_systems=['*'], + valid_prog_environs=['+foo -bar'], + expected={ + 'testsys:gpu': [], + 'testsys:login': [] + } + ) + # Check valid_systems / valid_prog_environs combinations _assert_supported( valid_systems=['testsys:login'],